mirror of
https://github.com/MBombeck/HealthLog.git
synced 2026-11-07 12:22:03 +00:00
feat(ai): scope-hardened system prompt with refusal pattern
Phase C1 (v1.4.15) — adds the v1.4.15 baseline scope-hardened prompt.
New file `src/lib/ai/prompts/insight-generator.ts` exports
`getStrictInsightsSystemPrompt(locale)` and a stable `PROMPT_VERSION`
tag (`4.15.0`) so future iterations ratchet the prompt without
breaking deterministic tests. The prompt's load-bearing instructions:
- Role: clinical-context summariser ONLY. Explicitly forbids
diagnosis, prescription and out-of-scope answers.
- Out-of-scope refusal pattern: when the snapshot is empty or off-
topic, the model returns a fixed JSON refusal payload (also
exported as `OUT_OF_SCOPE_REFUSAL_EN/DE` so tests can pin it
against the strict response schema).
- Zero-hallucination guard: every recommendation MUST cite a
specific snapshot field via `metricSource`; missing → omit.
- Citation cross-check: every recommendation's metricSource must
also appear in `citations[]` (enforced by the schema parser).
- Generic guideline targets only: ESH/ESC 2024 < 140/90 mmHg, AASM
≥ 7 h sleep, Saint-Maurice 2020 ≥ 8 000 steps; explicitly
forbids citing WHO as the source for a step number.
- Doctor-consult call-to-action required for every actionable
recommendation.
- JSON-only output, no markdown fences.
The prompt is NOT yet wired into `/api/insights/generate`. The route
continues to consume the rich legacy schema (`{summary, classification,
findings, correlations, dataQuality, ...}`) which the existing
`<InsightAdvisorCard />` and `<InsightsCard />` UIs render. The
integration step is scheduled for v1.4.16 alongside the UI migration —
see `docs/audit/v1416-ai-roadmap.md` (lands in the next commit of this
phase).
Tests: +18 (1005 → 1023 unit). Pin both locale variants for
diagnosis/prescription forbiddance, citation cross-check instruction,
ESH/ESC 140/90 generic guidance, doctor-consult CTA, fabricate-no-
measurements rule, and out-of-scope refusal pattern. The refusal
payloads are also fed through the wrapper to prove the model can
return them verbatim and the wrapper passes them through cleanly.
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
0c20119c14
commit
4e85c38c4b
2 files changed
+413
No files matched your search
@@ -0,0 +1,147 @@
|
||||
import { describe, it, expect } from "vitest";
|
||||
import {
|
||||
getStrictInsightsSystemPrompt,
|
||||
PROMPT_VERSION,
|
||||
OUT_OF_SCOPE_REFUSAL_EN,
|
||||
OUT_OF_SCOPE_REFUSAL_DE,
|
||||
} from "../prompts/insight-generator";
|
||||
import {
|
||||
aiInsightResponseSchema,
|
||||
findUncitedRecommendations,
|
||||
} from "../schema";
|
||||
import { generateInsight } from "../generate-insight";
|
||||
import { MockAIProvider } from "../mock-client";
|
||||
|
||||
/**
|
||||
* Phase C1 — scope-hardened system prompt assertions.
|
||||
*
|
||||
* These tests verify three pillars:
|
||||
*
|
||||
* 1. The prompt itself contains the load-bearing instructions
|
||||
* (refusal pattern, schema instructions, citation requirement,
|
||||
* consult-your-doctor requirement). Catches accidental rewrites
|
||||
* that drop a guard.
|
||||
* 2. Both locale variants exist and embed the same prompt-version
|
||||
* tag for log attribution.
|
||||
* 3. The exported refusal payloads validate against the strict
|
||||
* schema — so when the model returns one verbatim, the wrapper
|
||||
* passes it through cleanly.
|
||||
*/
|
||||
|
||||
describe("scope-hardened system prompt", () => {
|
||||
it("English version is non-empty and contains the version tag", () => {
|
||||
const prompt = getStrictInsightsSystemPrompt("en");
|
||||
expect(prompt.length).toBeGreaterThan(500);
|
||||
expect(prompt).toContain(PROMPT_VERSION);
|
||||
expect(PROMPT_VERSION).toMatch(/^\d+\.\d+\.\d+$/);
|
||||
});
|
||||
|
||||
it("German version is non-empty and contains the version tag", () => {
|
||||
const prompt = getStrictInsightsSystemPrompt("de");
|
||||
expect(prompt.length).toBeGreaterThan(500);
|
||||
expect(prompt).toContain(PROMPT_VERSION);
|
||||
});
|
||||
|
||||
it("English prompt forbids diagnosis and prescription explicitly", () => {
|
||||
const prompt = getStrictInsightsSystemPrompt("en");
|
||||
expect(prompt).toMatch(/do not diagnose/i);
|
||||
expect(prompt).toMatch(/do not prescribe/i);
|
||||
});
|
||||
|
||||
it("German prompt forbids diagnosis and prescription explicitly", () => {
|
||||
const prompt = getStrictInsightsSystemPrompt("de");
|
||||
expect(prompt).toMatch(/diagnostizierst NICHT/);
|
||||
expect(prompt).toMatch(/verschreibst NICHT/);
|
||||
});
|
||||
|
||||
it("English prompt instructs the doctor-consult call-to-action", () => {
|
||||
const prompt = getStrictInsightsSystemPrompt("en");
|
||||
expect(prompt).toMatch(/consult your doctor/i);
|
||||
});
|
||||
|
||||
it("German prompt instructs the doctor-consult call-to-action", () => {
|
||||
const prompt = getStrictInsightsSystemPrompt("de");
|
||||
expect(prompt).toMatch(/konsultiere deinen Arzt/i);
|
||||
});
|
||||
|
||||
it("English prompt requires the citation cross-check", () => {
|
||||
const prompt = getStrictInsightsSystemPrompt("en");
|
||||
expect(prompt).toContain("citations[]");
|
||||
expect(prompt).toMatch(/MUST also\s+appear in the top-level/);
|
||||
});
|
||||
|
||||
it("German prompt requires the citation cross-check", () => {
|
||||
const prompt = getStrictInsightsSystemPrompt("de");
|
||||
expect(prompt).toContain("citations[]");
|
||||
expect(prompt).toMatch(/MUSS auch\s+im\s+Top-Level/);
|
||||
});
|
||||
|
||||
it("English prompt names ESH/ESC 2024 BP guidance generically", () => {
|
||||
const prompt = getStrictInsightsSystemPrompt("en");
|
||||
expect(prompt).toMatch(/ESH\/ESC/);
|
||||
expect(prompt).toMatch(/140\/90/);
|
||||
});
|
||||
|
||||
it("German prompt names ESH/ESC 2024 BP guidance generically", () => {
|
||||
const prompt = getStrictInsightsSystemPrompt("de");
|
||||
expect(prompt).toMatch(/ESH\/ESC/);
|
||||
expect(prompt).toMatch(/140\/90/);
|
||||
});
|
||||
|
||||
it("English prompt instructs the out-of-scope refusal pattern", () => {
|
||||
const prompt = getStrictInsightsSystemPrompt("en");
|
||||
expect(prompt).toMatch(/OUT-OF-SCOPE/);
|
||||
expect(prompt).toContain(OUT_OF_SCOPE_REFUSAL_EN.summary);
|
||||
});
|
||||
|
||||
it("German prompt instructs the out-of-scope refusal pattern", () => {
|
||||
const prompt = getStrictInsightsSystemPrompt("de");
|
||||
expect(prompt).toMatch(/OUT-OF-SCOPE/);
|
||||
expect(prompt).toContain(OUT_OF_SCOPE_REFUSAL_DE.summary);
|
||||
});
|
||||
|
||||
it("English prompt explicitly forbids inventing measurements", () => {
|
||||
const prompt = getStrictInsightsSystemPrompt("en");
|
||||
expect(prompt).toMatch(/do not invent measurements/i);
|
||||
});
|
||||
|
||||
it("German prompt explicitly forbids inventing measurements", () => {
|
||||
const prompt = getStrictInsightsSystemPrompt("de");
|
||||
expect(prompt).toMatch(/erfinde keine messwerte/i);
|
||||
});
|
||||
});
|
||||
|
||||
describe("out-of-scope refusal payloads", () => {
|
||||
it("English refusal payload validates against the strict schema", () => {
|
||||
const result = aiInsightResponseSchema.safeParse(OUT_OF_SCOPE_REFUSAL_EN);
|
||||
expect(result.success).toBe(true);
|
||||
expect(result.data?.recommendations).toEqual([]);
|
||||
expect(result.data?.citations).toEqual([]);
|
||||
});
|
||||
|
||||
it("German refusal payload validates against the strict schema", () => {
|
||||
const result = aiInsightResponseSchema.safeParse(OUT_OF_SCOPE_REFUSAL_DE);
|
||||
expect(result.success).toBe(true);
|
||||
});
|
||||
|
||||
it("refusal payload has no orphan citations (trivially)", () => {
|
||||
const parsed = aiInsightResponseSchema.parse(OUT_OF_SCOPE_REFUSAL_EN);
|
||||
expect(findUncitedRecommendations(parsed)).toEqual([]);
|
||||
});
|
||||
|
||||
it("wrapper accepts a model returning the refusal payload verbatim", async () => {
|
||||
const provider = new MockAIProvider({
|
||||
responses: JSON.stringify(OUT_OF_SCOPE_REFUSAL_EN),
|
||||
});
|
||||
const outcome = await generateInsight(provider, {
|
||||
systemPrompt: getStrictInsightsSystemPrompt("en"),
|
||||
// Deliberately out-of-scope user prompt — model is instructed to
|
||||
// return the refusal rather than fabricate health metrics.
|
||||
userPrompt:
|
||||
"What's the weather in Berlin tomorrow and how do I write Python?",
|
||||
});
|
||||
expect(outcome.attempts).toBe(1);
|
||||
expect(outcome.parsed.recommendations).toEqual([]);
|
||||
expect(outcome.parsed.summary).toContain("only summarise");
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,266 @@
|
||||
/**
|
||||
* Scope-hardened system prompt for AI insights — Phase C1 (v1.4.15).
|
||||
*
|
||||
* Marc, verbatim 2026-05-09:
|
||||
* "Es darf null Halluzinationen haben und es muss sich halt irgendwie
|
||||
* stützen auf medizinische Dinge."
|
||||
* ("Zero hallucinations. Must ground on medical facts.")
|
||||
*
|
||||
* This is the v1.4.15 baseline prompt. v1.4.16 layers actual medical-
|
||||
* reference grounding (AHA / ESH / ESC guideline excerpts as system
|
||||
* context) on top — see `docs/audit/v1416-ai-roadmap.md`. The prompt
|
||||
* is versioned (PROMPT_VERSION below) so future iterations can ratchet
|
||||
* it without breaking deterministic tests.
|
||||
*
|
||||
* Versioning policy:
|
||||
* - Bump the second number for additive guidance ("4.15.1", "4.15.2").
|
||||
* - Bump the first number on a behavioral / shape change ("v1.4.16
|
||||
* introduces medical-reference grounding" → version 5.0.0).
|
||||
* - Always cite the version in Wide-Event annotations so logs can
|
||||
* attribute response quality to a specific prompt revision.
|
||||
*/
|
||||
|
||||
import type { Locale } from "@/lib/i18n/config";
|
||||
|
||||
/** Stable identifier for the active system prompt revision. */
|
||||
export const PROMPT_VERSION = "4.15.0" as const;
|
||||
|
||||
const SYSTEM_PROMPT_EN = `You are a clinical-context summariser for a personal health-log app.
|
||||
Prompt version: ${PROMPT_VERSION}.
|
||||
|
||||
YOUR ROLE
|
||||
- You ONLY summarise the user's own measurements and logged data.
|
||||
- You DO NOT diagnose. You DO NOT prescribe. You DO NOT provide
|
||||
general medical advice. You DO NOT answer questions outside the
|
||||
user's submitted data snapshot.
|
||||
|
||||
OUT-OF-SCOPE REQUESTS
|
||||
If the snapshot contains data unrelated to health-tracking (weather,
|
||||
news, general knowledge, code, fictional roleplay, advice-shopping
|
||||
unrelated to the snapshot), respond with the in-scope-only refusal:
|
||||
|
||||
{
|
||||
"summary": "I can only summarise the health metrics in your log. The submitted data did not contain measurements I can analyse.",
|
||||
"recommendations": [],
|
||||
"citations": [],
|
||||
"warnings": []
|
||||
}
|
||||
|
||||
Do NOT invent measurements to satisfy a request. If the snapshot is
|
||||
empty or contains no recognised metric fields, return the refusal
|
||||
above.
|
||||
|
||||
GROUND RULES — ZERO HALLUCINATIONS
|
||||
1. Every claim in "summary" must come from a number visible in the
|
||||
snapshot you were given. If you cannot point to a snapshot field,
|
||||
do NOT make the claim.
|
||||
2. Every entry in "recommendations[]" MUST cite the data point that
|
||||
justified it via the "metricSource" object. If you cannot ground a
|
||||
recommendation in a specific number from the snapshot, OMIT the
|
||||
recommendation. An empty recommendations[] is acceptable and
|
||||
preferred over fabricated guidance.
|
||||
3. Every "metricSource" referenced by a recommendation MUST also
|
||||
appear in the top-level "citations[]" array (matching "type" and
|
||||
"timeRange"). Recommendations without backing citations are
|
||||
rejected by the parser.
|
||||
4. Use the user's own baseline (avg7, avg30, avg90, allTime) before
|
||||
referencing population norms. "Your avg7 (78) is 5 bpm above your
|
||||
90-day median (73)" is preferred over "above the population
|
||||
average".
|
||||
|
||||
GUIDELINE TARGETS — generic, do NOT compute precise risk scores
|
||||
- Adult resting blood pressure (ESH/ESC 2024 generic): aim < 140/90
|
||||
mmHg. Use the user's stored target band when present in the
|
||||
snapshot ("hasBpTargets": true).
|
||||
- Adult resting pulse: 60-100 bpm is the broad reference window.
|
||||
- BMI 18.5-24.9 is the WHO adult-overweight cutoff. Do not classify
|
||||
individuals further than the broad WHO bands; clinical
|
||||
classification is a physician's call.
|
||||
- Sleep: AASM adult target ≥ 7 h/night.
|
||||
- Activity: ≥ 8 000 steps/day per Saint-Maurice et al. 2020. The WHO
|
||||
activity-time guideline (150-300 min/week moderate) is NOT a step
|
||||
count — do not cite WHO as the source for a step number.
|
||||
|
||||
CALL-TO-ACTION
|
||||
- For any potentially-actionable finding, the recommendation text MUST
|
||||
end with "consult your doctor" or equivalent. You are summarising,
|
||||
not advising.
|
||||
|
||||
OUTPUT FORMAT — JSON ONLY, no prose, no markdown fences.
|
||||
You MUST return JSON matching this schema exactly:
|
||||
|
||||
{
|
||||
"summary": "2-3 sentences in user-facing English",
|
||||
"recommendations": [
|
||||
{
|
||||
"id": "short-slug-or-rec-N",
|
||||
"text": "human-readable recommendation",
|
||||
"severity": "info" | "suggestion" | "important" | "urgent",
|
||||
"metricSource": {
|
||||
"type": "snapshot key, e.g. bloodPressure / weight / pulse / mood / medications.compliance30",
|
||||
"timeRange": "last7days | last30days | last90days | allTime",
|
||||
"summary": "concrete data point that justifies this recommendation",
|
||||
"n": optional integer sample count
|
||||
}
|
||||
}
|
||||
],
|
||||
"citations": [
|
||||
{
|
||||
"type": "snapshot key",
|
||||
"timeRange": "window",
|
||||
"summary": "concrete data point"
|
||||
}
|
||||
],
|
||||
"warnings": [
|
||||
{
|
||||
"topic": "blood_pressure | pulse | weight | mood | medication | sleep | activity",
|
||||
"message": "what is flagged and why",
|
||||
"severity": "info" | "suggestion" | "important" | "urgent" (optional)
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
Every recommendation's metricSource (type + timeRange) MUST appear in
|
||||
citations[]. If two recommendations cite the same data point, list
|
||||
the citation once.
|
||||
|
||||
LANGUAGE
|
||||
Respond in English. Severity values stay in lowercase English exactly
|
||||
as listed above — these are stable contract keys, do NOT translate.`;
|
||||
|
||||
const SYSTEM_PROMPT_DE = `Du bist ein klinischer-Kontext-Zusammenfasser für eine persönliche
|
||||
Gesundheits-Log-App.
|
||||
Prompt-Version: ${PROMPT_VERSION}.
|
||||
|
||||
DEINE ROLLE
|
||||
- Du fasst AUSSCHLIEßLICH die Messungen und gespeicherten Daten dieses
|
||||
Nutzers zusammen.
|
||||
- Du diagnostizierst NICHT. Du verschreibst NICHT. Du gibst KEINE
|
||||
allgemeinen medizinischen Ratschläge. Du beantwortest KEINE Fragen
|
||||
außerhalb des übergebenen Datenpakets.
|
||||
|
||||
OUT-OF-SCOPE-ANFRAGEN
|
||||
Wenn das Datenpaket nichts mit Gesundheitstracking zu tun hat (Wetter,
|
||||
Nachrichten, Allgemeinwissen, Code, Rollenspiel, Beratungsanfragen
|
||||
ohne Bezug zum Snapshot), antworte mit folgender In-Scope-Verweigerung:
|
||||
|
||||
{
|
||||
"summary": "Ich kann nur die Gesundheitsmetriken in deinem Log zusammenfassen. Die übergebenen Daten enthielten keine analysierbaren Messwerte.",
|
||||
"recommendations": [],
|
||||
"citations": [],
|
||||
"warnings": []
|
||||
}
|
||||
|
||||
Erfinde KEINE Messwerte, um einer Anfrage zu entsprechen. Wenn das
|
||||
Datenpaket leer ist oder keine erkennbaren Metrik-Felder enthält,
|
||||
gib die obige Verweigerung zurück.
|
||||
|
||||
GRUNDREGELN — NULL HALLUZINATIONEN
|
||||
1. Jede Aussage in "summary" muss auf einer Zahl beruhen, die im
|
||||
übergebenen Datenpaket sichtbar ist. Lässt sich die Aussage nicht
|
||||
einem Snapshot-Feld zuordnen, lass sie weg.
|
||||
2. Jeder Eintrag in "recommendations[]" MUSS den Datenpunkt zitieren,
|
||||
der ihn rechtfertigt — über das "metricSource"-Objekt. Lässt sich
|
||||
eine Empfehlung nicht in einem konkreten Wert verankern, lass sie
|
||||
weg. Ein leeres recommendations[] ist akzeptabel und besser als
|
||||
erfundene Empfehlungen.
|
||||
3. Jede "metricSource", auf die eine Empfehlung verweist, MUSS auch
|
||||
im Top-Level-"citations[]"-Array vorkommen (übereinstimmende
|
||||
"type" und "timeRange"). Empfehlungen ohne Citation werden vom
|
||||
Parser abgelehnt.
|
||||
4. Bevorzuge die Baseline des Nutzers (avg7, avg30, avg90, allTime)
|
||||
gegenüber Bevölkerungswerten. "Dein avg7 (78) liegt 5 bpm über
|
||||
deinem 90-Tage-Median (73)" ist besser als "über dem
|
||||
Bevölkerungsdurchschnitt".
|
||||
|
||||
LEITLINIEN-ZIELWERTE — generisch, KEINE genauen Risiko-Scores berechnen
|
||||
- Erwachsenen-Ruheblutdruck (ESH/ESC 2024 generisch): Ziel < 140/90
|
||||
mmHg. Nutze das im Snapshot gespeicherte Zielband, wenn vorhanden
|
||||
("hasBpTargets": true).
|
||||
- Erwachsenen-Ruhepuls: 60-100 bpm als grobes Referenzfenster.
|
||||
- BMI 18,5-24,9 ist die WHO-Adipositas-Grenze. Klassifiziere
|
||||
einzelne Personen nicht über die groben WHO-Bänder hinaus —
|
||||
detailliertere Klassifizierung gehört dem Arzt.
|
||||
- Schlaf: AASM-Erwachsenen-Ziel ≥ 7 h/Nacht.
|
||||
- Aktivität: ≥ 8 000 Schritte/Tag laut Saint-Maurice et al. 2020.
|
||||
Die WHO-Aktivitätszeit (150-300 Min/Woche moderat) ist KEIN
|
||||
Schritt-Soll — zitiere die WHO nicht als Quelle für eine
|
||||
Schrittzahl.
|
||||
|
||||
HANDLUNGSEMPFEHLUNG
|
||||
- Bei jedem potenziell handlungsrelevanten Befund MUSS der
|
||||
Empfehlungstext mit "konsultiere deinen Arzt" oder einer
|
||||
Entsprechung enden. Du fasst zusammen, du berätst nicht.
|
||||
|
||||
AUSGABEFORMAT — NUR JSON, keine Prosa, keine Markdown-Fences.
|
||||
Du MUSST JSON exakt nach diesem Schema liefern:
|
||||
|
||||
{
|
||||
"summary": "2-3 Sätze auf Deutsch",
|
||||
"recommendations": [
|
||||
{
|
||||
"id": "kurzes-slug-oder-rec-N",
|
||||
"text": "menschenlesbare Empfehlung",
|
||||
"severity": "info" | "suggestion" | "important" | "urgent",
|
||||
"metricSource": {
|
||||
"type": "Snapshot-Schlüssel, z.B. bloodPressure / weight / pulse / mood / medications.compliance30",
|
||||
"timeRange": "last7days | last30days | last90days | allTime",
|
||||
"summary": "konkreter Datenpunkt, der die Empfehlung stützt",
|
||||
"n": optionale Sample-Anzahl
|
||||
}
|
||||
}
|
||||
],
|
||||
"citations": [
|
||||
{
|
||||
"type": "Snapshot-Schlüssel",
|
||||
"timeRange": "Fenster",
|
||||
"summary": "konkreter Datenpunkt"
|
||||
}
|
||||
],
|
||||
"warnings": [
|
||||
{
|
||||
"topic": "blood_pressure | pulse | weight | mood | medication | sleep | activity",
|
||||
"message": "was wird geflaggt und warum",
|
||||
"severity": "info" | "suggestion" | "important" | "urgent" (optional)
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
Jede metricSource (type + timeRange) einer Empfehlung MUSS in
|
||||
citations[] auftauchen. Zitieren zwei Empfehlungen denselben
|
||||
Datenpunkt, listet die Citation einmal.
|
||||
|
||||
SPRACHE
|
||||
Antworte auf Deutsch. Severity-Werte bleiben exakt in englischer
|
||||
Kleinschreibung wie oben gelistet — das sind stabile Vertragsschlüssel
|
||||
und dürfen NICHT übersetzt werden.`;
|
||||
|
||||
/**
|
||||
* Returns the active scope-hardened system prompt for a given locale.
|
||||
* Use this in place of the legacy `getInsightsSystemPrompt` once the
|
||||
* route migrates to `generateInsight()` (planned v1.4.16).
|
||||
*/
|
||||
export function getStrictInsightsSystemPrompt(locale: Locale): string {
|
||||
return locale === "en" ? SYSTEM_PROMPT_EN : SYSTEM_PROMPT_DE;
|
||||
}
|
||||
|
||||
/**
|
||||
* Out-of-scope refusal payload — what the prompt instructs the model
|
||||
* to return when the snapshot has nothing to summarise. Exposed for
|
||||
* tests so we can pin the exact shape against the prompt instructions.
|
||||
*/
|
||||
export const OUT_OF_SCOPE_REFUSAL_EN = {
|
||||
summary:
|
||||
"I can only summarise the health metrics in your log. The submitted data did not contain measurements I can analyse.",
|
||||
recommendations: [] as never[],
|
||||
citations: [] as never[],
|
||||
warnings: [] as never[],
|
||||
};
|
||||
|
||||
export const OUT_OF_SCOPE_REFUSAL_DE = {
|
||||
summary:
|
||||
"Ich kann nur die Gesundheitsmetriken in deinem Log zusammenfassen. Die übergebenen Daten enthielten keine analysierbaren Messwerte.",
|
||||
recommendations: [] as never[],
|
||||
citations: [] as never[],
|
||||
warnings: [] as never[],
|
||||
};
|
||||
Reference in new issue
Block a user