feat(coach): parse evidence-block sentinel out of streamed prose

Add a text-marker sentinel parser (`---KEYVALUES---` … `---END---`)
that strips the Coach's evidence block out of the prose before it
streams to the client, validates each line against a Zod schema, and
attaches the parsed entries to `provenance.keyValues`. The persisted
envelope round-trips so the disclosure renders on conversation
reload. Hard-capped at 8 lines and 1 KB to guard against
prompt-injection padding; malformed blocks log a wide-event and pass
the prose through unchanged.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
Marc-André BombeckandClaude Opus 4.7 committed 2026-05-10 21:07:32 +02:00
1 parent bea55eac5c
commit 7b8efaae14
6 files changed
+579 -8

No files matched your search

+29 -4
View File
@@ -60,6 +60,7 @@ import {
import { detectRefusal } from "@/lib/ai/coach/refusal";
import { getCoachSystemPrompt } from "@/lib/ai/coach/system-prompt";
import { buildCoachSnapshot } from "@/lib/ai/coach/snapshot";
import { parseKeyValuesSentinel } from "@/lib/ai/coach/keyvalues";
/**
* Hard cap on total turns kept inside the per-call prompt window.
@@ -260,18 +261,42 @@ Reply now as the assistant, in ${locale === "de" ? "German" : "English"}.`;
throw err;
}
const replyText = (result.content ?? "").trim();
if (!replyText) {
const rawReply = (result.content ?? "").trim();
if (!rawReply) {
return streamProviderError({ code: "coach.provider.empty" });
}
// v1.4.22 — strip the optional `---KEYVALUES---` … `---END---`
// sentinel out of the prose. The stripped prose is what we stream
// to the client and persist; the parsed entries enrich the
// provenance envelope so the UI can render the collapsible
// "Worauf bezieht sich das?" disclosure.
const sentinel = parseKeyValuesSentinel(rawReply);
const replyText = sentinel.prose.trim() || rawReply;
const enrichedProvenance =
sentinel.keyValues.length > 0
? { ...snapshot.provenance, keyValues: sentinel.keyValues }
: snapshot.provenance;
if (sentinel.malformed) {
// Graceful degrade: log so ops can spot a provider whose
// sentinel format has drifted, but pass the prose through
// unchanged.
annotate({
action: { name: "coach.keyvalues.parse_failed" },
meta: {
kept: sentinel.keyValues.length,
promptVersion: PROMPT_VERSION,
},
});
}
// Persist the assistant message BEFORE we begin streaming; if the
// client disconnects we still have the canonical row.
const assistantMessage = await appendMessage({
conversationId: workingConversationId,
role: "assistant",
content: replyText,
metricSource: snapshot.provenance,
metricSource: enrichedProvenance,
providerType: workingProviderType,
promptVersion: PROMPT_VERSION,
});
@@ -305,7 +330,7 @@ Reply now as the assistant, in ${locale === "de" ? "German" : "English"}.`;
controller.enqueue(
encodeFrame({
type: "provenance",
metricSource: snapshot.provenance,
metricSource: enrichedProvenance,
}),
);
controller.enqueue(
@@ -0,0 +1,180 @@
import { describe, expect, it } from "vitest";
import {
parseKeyValueLine,
parseKeyValuesSentinel,
} from "../keyvalues";
/**
* v1.4.22 Wave 3 B1+B2 — Coach evidence-block sentinel parser.
*/
describe("parseKeyValueLine", () => {
it("parses a full line with unit + window", () => {
expect(parseKeyValueLine("avg30 systolic: 138 [mmHg] (last30days)")).toEqual({
label: "avg30 systolic",
value: "138",
unit: "mmHg",
window: "last30days",
});
});
it("parses a line with unit but no window (day-level pin)", () => {
expect(parseKeyValueLine("Tue 6 May: 142/88 [mmHg]")).toEqual({
label: "Tue 6 May",
value: "142/88",
unit: "mmHg",
});
});
it("parses a line with neither unit nor window", () => {
expect(parseKeyValueLine("30-day adherence: 96")).toEqual({
label: "30-day adherence",
value: "96",
});
});
it("tolerates extra whitespace around tokens", () => {
expect(
parseKeyValueLine(" avg30 systolic : 138 [mmHg] (last30days) "),
).toEqual({
label: "avg30 systolic",
value: "138",
unit: "mmHg",
window: "last30days",
});
});
it("returns null for an empty line", () => {
expect(parseKeyValueLine("")).toBeNull();
expect(parseKeyValueLine(" ")).toBeNull();
});
it("returns null for a line with no colon", () => {
expect(parseKeyValueLine("no separator here")).toBeNull();
});
it("returns null when the label is empty", () => {
expect(parseKeyValueLine(": 138")).toBeNull();
});
it("returns null when the value would be empty", () => {
expect(parseKeyValueLine("label only:")).toBeNull();
});
});
describe("parseKeyValuesSentinel", () => {
it("strips the sentinel block out of the prose", () => {
const raw = [
"Your last week sits a touch higher than your usual run.",
"",
"---KEYVALUES---",
"avg7 systolic: 138 [mmHg] (last7days)",
"avg30 systolic: 134 [mmHg] (last30days)",
"---END---",
].join("\n");
const out = parseKeyValuesSentinel(raw);
expect(out.prose).toBe(
"Your last week sits a touch higher than your usual run.",
);
expect(out.keyValues).toEqual([
{
label: "avg7 systolic",
value: "138",
unit: "mmHg",
window: "last7days",
},
{
label: "avg30 systolic",
value: "134",
unit: "mmHg",
window: "last30days",
},
]);
expect(out.malformed).toBe(false);
});
it("returns prose untouched when no sentinel is present", () => {
const out = parseKeyValuesSentinel(
"Just a qualitative reply with no numbers cited.",
);
expect(out.prose).toBe("Just a qualitative reply with no numbers cited.");
expect(out.keyValues).toEqual([]);
expect(out.malformed).toBe(false);
});
it("graceful-degrades when the closing ---END--- is missing", () => {
const raw = [
"Prose still streams cleanly.",
"",
"---KEYVALUES---",
"avg7 systolic: 138 [mmHg] (last7days)",
].join("\n");
const out = parseKeyValuesSentinel(raw);
// Prose stripped, body parsed, but flagged malformed so the route
// can log a wide-event for ops visibility.
expect(out.prose).toBe("Prose still streams cleanly.");
expect(out.keyValues.length).toBe(1);
expect(out.malformed).toBe(true);
});
it("clips the block to 8 entries (prompt-contract cap)", () => {
const lines = Array.from(
{ length: 12 },
(_, i) => `metric-${i}: ${100 + i} [mmHg] (last30days)`,
);
const raw = ["---KEYVALUES---", ...lines, "---END---"].join("\n");
const out = parseKeyValuesSentinel(raw);
expect(out.keyValues.length).toBe(8);
expect(out.keyValues[0].label).toBe("metric-0");
expect(out.keyValues[7].label).toBe("metric-7");
});
it("clips an oversized block at the 1 KB cap", () => {
// 1 200 bytes of label/value pairs — every line passes the regex,
// but the parser trims to 1 024 bytes before splitting so we end
// up with whichever rows fit.
const padded = Array.from(
{ length: 60 },
(_, i) => `label-${i}: ${i.toString().padEnd(15, "x")}`,
);
const raw = ["---KEYVALUES---", ...padded, "---END---"].join("\n");
const out = parseKeyValuesSentinel(raw);
expect(out.keyValues.length).toBeLessThanOrEqual(8);
expect(out.malformed).toBe(true);
});
it("silently drops malformed lines mixed in with valid ones", () => {
const raw = [
"Prose.",
"",
"---KEYVALUES---",
"avg30 systolic: 138 [mmHg] (last30days)",
"not a valid line",
":no-label-here",
"avg30 mood: 4.1 [/5] (last30days)",
"---END---",
].join("\n");
const out = parseKeyValuesSentinel(raw);
expect(out.keyValues.length).toBe(2);
expect(out.keyValues.map((k) => k.label)).toEqual([
"avg30 systolic",
"avg30 mood",
]);
});
it("returns malformed when the sentinel block is empty", () => {
const raw = ["Prose.", "---KEYVALUES---", "---END---"].join("\n");
const out = parseKeyValuesSentinel(raw);
expect(out.keyValues).toEqual([]);
expect(out.malformed).toBe(true);
expect(out.prose).toBe("Prose.");
});
it("returns prose and an empty array for empty input", () => {
const out = parseKeyValuesSentinel("");
expect(out.prose).toBe("");
expect(out.keyValues).toEqual([]);
expect(out.malformed).toBe(false);
});
});
+180
View File
@@ -0,0 +1,180 @@
/**
* v1.4.22 — Coach evidence-block (`---KEYVALUES---` … `---END---`)
* parser.
*
* The Coach prompt instructs the model to append, after its prose
* reply, a sentinel block carrying the load-bearing numbers it cited:
*
* ---KEYVALUES---
* avg30 systolic: 138 [mmHg] (last30days)
* Tue 6 May: 142/88 [mmHg]
* ---END---
*
* This module strips that block out of the streamed prose and parses
* each line into a `CoachKeyValue`. The result feeds the
* `provenance.keyValues` field that the UI renders as a collapsible
* disclosure.
*
* Defence-in-depth:
* - Hard cap 1 KB on the sentinel block (prompt-injection guard).
* - Hard cap 8 lines kept (per the prompt contract).
* - Each line is validated against `coachKeyValueSchema`; malformed
* lines are dropped silently.
* - When the sentinel pair is malformed (e.g. no closing `---END---`,
* unbalanced sentinels), the parser returns the original prose
* untouched and emits no keyValues — the caller logs a wide-event
* `coach.keyvalues.parse_failed` for ops visibility.
*/
import { coachKeyValueSchema, type CoachKeyValue } from "./types";
/** Sentinel block payload hard cap — 1 KB after the opening marker. */
const SENTINEL_BYTE_CAP = 1024;
/** Max key/value rows kept inside the block (prompt-contract value). */
const SENTINEL_LINE_CAP = 8;
const OPEN_SENTINEL = "---KEYVALUES---";
const CLOSE_SENTINEL = "---END---";
/**
* Parse one body line of the form
* `<label>: <value> [<unit>] (<window>)`
* into a `CoachKeyValue`. The unit + window are both optional; an
* input missing either or both still parses cleanly. Returns `null`
* when the line does not match the expected shape — the caller drops
* the line rather than rejecting the whole block.
*/
export function parseKeyValueLine(line: string): CoachKeyValue | null {
const trimmed = line.trim();
if (!trimmed) return null;
// Find the first `:` — the label cannot itself contain `:`. The
// remainder may carry the value, an optional `[unit]`, and an
// optional `(window)`.
const colon = trimmed.indexOf(":");
if (colon < 1) return null;
const label = trimmed.slice(0, colon).trim();
let rest = trimmed.slice(colon + 1).trim();
if (!label || !rest) return null;
let unit: string | undefined;
let window: string | undefined;
// Extract (window) if present at the very end. We allow the unit
// bracket to follow either the value or the window-paren — both
// emit orders show up across English/German few-shot examples.
const winMatch = rest.match(/\s*\(([^()]+)\)\s*$/);
if (winMatch) {
window = winMatch[1].trim();
rest = rest.slice(0, winMatch.index).trim();
}
// Extract [unit] if present at the end (after the optional window
// strip).
const unitMatch = rest.match(/\s*\[([^[\]]+)\]\s*$/);
if (unitMatch) {
unit = unitMatch[1].trim();
rest = rest.slice(0, unitMatch.index).trim();
}
const value = rest.trim();
if (!value) return null;
const parsed = coachKeyValueSchema.safeParse({
label,
value,
...(unit ? { unit } : {}),
...(window ? { window } : {}),
});
if (!parsed.success) return null;
return parsed.data;
}
export interface SentinelParseResult {
/** Prose with the sentinel block (if any) stripped out. */
prose: string;
/** Parsed key/value rows. Empty when no valid sentinel was found. */
keyValues: CoachKeyValue[];
/**
* `true` when the parser saw an opening `---KEYVALUES---` marker
* but failed to extract a complete + non-empty + well-formed block.
* Callers log this for ops visibility (graceful degrade).
*/
malformed: boolean;
}
/**
* Split a raw assistant reply into `{ prose, keyValues }`.
*
* The streamed prose passed to the client is the `prose` field — the
* `---KEYVALUES---` block is stripped before any token frame is sent
* so the user never sees the raw sentinel.
*/
export function parseKeyValuesSentinel(raw: string): SentinelParseResult {
if (!raw) {
return { prose: "", keyValues: [], malformed: false };
}
const openIdx = raw.indexOf(OPEN_SENTINEL);
if (openIdx === -1) {
return { prose: raw, keyValues: [], malformed: false };
}
// Everything before the opening marker is prose; trim trailing
// whitespace introduced by the marker's own leading newline so the
// bubble doesn't render an extra blank paragraph.
const prose = raw.slice(0, openIdx).replace(/\s+$/u, "");
const afterOpen = raw.slice(openIdx + OPEN_SENTINEL.length);
// Look for the closing sentinel; if missing, the block is
// malformed — fall back to the original prose and emit nothing.
const closeRel = afterOpen.indexOf(CLOSE_SENTINEL);
let bodyRaw: string;
let malformedClose = false;
if (closeRel === -1) {
// Treat the rest of the reply as the block body so we can still
// parse what's there — but flag malformed so the caller knows
// the contract wasn't fully met.
bodyRaw = afterOpen;
malformedClose = true;
} else {
bodyRaw = afterOpen.slice(0, closeRel);
}
// Defence-in-depth: cap the sentinel payload at 1 KB before we
// bother parsing. An adversarial reply could otherwise pad the
// block with megabytes of garbage.
let body = bodyRaw;
let truncated = false;
if (body.length > SENTINEL_BYTE_CAP) {
body = body.slice(0, SENTINEL_BYTE_CAP);
truncated = true;
}
const lines = body
.split(/\r?\n/)
.map((l) => l.trim())
.filter((l) => l.length > 0)
// Skip any stray opening / closing markers that landed inside the
// body (rare but cheap to ignore).
.filter((l) => l !== OPEN_SENTINEL && l !== CLOSE_SENTINEL);
const kept: CoachKeyValue[] = [];
for (const line of lines) {
if (kept.length >= SENTINEL_LINE_CAP) break;
const entry = parseKeyValueLine(line);
if (entry) kept.push(entry);
}
// Malformed when:
// - closing sentinel missing, OR
// - block payload was truncated to fit the 1 KB cap, OR
// - the block was present but yielded zero valid rows.
const malformed = malformedClose || truncated || kept.length === 0;
return {
prose,
keyValues: kept,
malformed,
};
}
+40 -1
View File
@@ -94,7 +94,46 @@ function provenanceFromJson(raw: string | null): CoachProvenance | null {
parsed.counts && typeof parsed.counts === "object"
? (parsed.counts as CoachProvenance["counts"])
: undefined;
return { windows, metrics, counts };
// v1.4.22 — keyValues are persisted alongside the existing
// windows/metrics/counts envelope so the evidence-block disclosure
// re-renders on conversation reload. Defensive shape check to
// tolerate legacy rows (no keyValues field) without throwing.
let keyValues: CoachProvenance["keyValues"];
if (Array.isArray(parsed.keyValues)) {
const cleaned: Array<{
label: string;
value: string;
unit?: string;
window?: string;
}> = [];
for (const raw of parsed.keyValues) {
if (!raw || typeof raw !== "object") continue;
const candidate = raw as {
label?: unknown;
value?: unknown;
unit?: unknown;
window?: unknown;
};
if (
typeof candidate.label !== "string" ||
typeof candidate.value !== "string"
) {
continue;
}
const entry: {
label: string;
value: string;
unit?: string;
window?: string;
} = { label: candidate.label, value: candidate.value };
if (typeof candidate.unit === "string") entry.unit = candidate.unit;
if (typeof candidate.window === "string")
entry.window = candidate.window;
cleaned.push(entry);
}
if (cleaned.length > 0) keyValues = cleaned;
}
return { windows, metrics, counts, ...(keyValues ? { keyValues } : {}) };
} catch {
return null;
}
+34 -3
View File
@@ -88,12 +88,36 @@ export type CoachStreamEvent =
| { type: "done"; conversationId: string; messageId: string }
| { type: "error"; code: string; message: string };
/**
* v1.4.22 — Zod schema for one entry inside the Coach's evidence
* block. The model emits these as lines between the
* `---KEYVALUES---` / `---END---` sentinels at the end of the reply;
* the route parses them out of the prose, attaches them to the
* provenance envelope, and the UI renders them inside the collapsible
* "Worauf bezieht sich das?" disclosure.
*
* Every field is length-capped so a malformed or adversarial sentinel
* cannot blow up the persisted payload. The whole block is also
* hard-capped to 8 entries / 1 KB by the parser.
*/
export const coachKeyValueSchema = z.object({
label: z.string().min(1).max(80),
value: z.string().min(1).max(40),
unit: z.string().max(16).optional(),
window: z.string().max(40).optional(),
});
export type CoachKeyValue = z.infer<typeof coachKeyValueSchema>;
/**
* Provenance envelope attached to assistant messages.
*
* NOTE: labels only — never raw values. The frontend renders source
* chips (e.g. "based on last30days BP, n=12") from these fields. Raw
* numbers and PII stay inside `encryptedContent`.
* NOTE: labels only — never raw values from the snapshot itself. The
* `keyValues` field added in v1.4.22 carries the load-bearing numbers
* the model chose to surface; those values come from the model's reply
* (which is itself grounded in the SNAPSHOT) and stay in the
* persisted `metricSourceJson` alongside the windows + metrics +
* counts so the disclosure can re-render on conversation reload.
*/
export interface CoachProvenance {
/**
@@ -115,6 +139,13 @@ export interface CoachProvenance {
counts?: Partial<
Record<"bp" | "weight" | "pulse" | "mood" | "compliance", number>
>;
/**
* v1.4.22 — load-bearing numbers the Coach drew on for this turn,
* surfaced in the collapsible evidence block under the assistant
* bubble. Optional; omit when the turn was qualitative or when the
* snapshot was empty. Hard cap 8 entries to keep the block scannable.
*/
keyValues?: ReadonlyArray<CoachKeyValue>;
}
/**
+116
View File
@@ -282,4 +282,120 @@ describe("POST /api/insights/chat — integration", () => {
);
expect(res.status).toBe(422);
});
it("strips the ---KEYVALUES--- block and surfaces keyValues in the persisted provenance", async () => {
// v1.4.22 — sentinel round-trip. The route should parse the
// model's evidence block out of the prose, persist it onto the
// assistant message's `metricSourceJson`, and emit it on the
// SSE `provenance` frame so the disclosure has data to render.
await seedUserWithSession();
runProviderMock.mockResolvedValue({
result: {
content: [
"The last week sits a touch higher than your usual run.",
"Anything different about this week — sleep, work, travel?",
"",
"---KEYVALUES---",
"avg7 systolic: 138 [mmHg] (last7days)",
"avg30 systolic: 134 [mmHg] (last30days)",
"---END---",
].join("\n"),
tokensUsed: 256,
model: "mock",
providerType: "openai",
},
workingProvider: { providerType: "openai", instance: {} },
fallbackHops: [],
});
const { POST } = await import("@/app/api/insights/chat/route");
const res = await (POST as (req: Request) => Promise<Response>)(
new Request("http://localhost/api/insights/chat", {
method: "POST",
headers: { "content-type": "application/json" },
body: JSON.stringify({
message: "How is my blood pressure trending this month?",
}),
}),
);
expect(res.status).toBe(200);
const text = await readStream(res);
// Streamed prose must NOT contain the raw sentinel markers.
const tokens: string[] = [];
for (const m of text.matchAll(/"type":"token","token":"([^"]*)"/g)) {
tokens.push(
m[1].replace(/\\n/g, "\n").replace(/\\"/g, '"').replace(/\\\\/g, "\\"),
);
}
const streamedProse = tokens.join("");
expect(streamedProse).toMatch(/sits a touch higher/);
expect(streamedProse).not.toMatch(/---KEYVALUES---/);
expect(streamedProse).not.toMatch(/---END---/);
// Provenance frame carries the parsed keyValues.
expect(text).toMatch(/"keyValues"/);
expect(text).toMatch(/avg7 systolic/);
// Persisted shape keeps the structured envelope.
const prisma = getPrismaClient();
const messages = await prisma.coachMessage.findMany({
orderBy: { createdAt: "asc" },
});
expect(messages.length).toBe(2);
const assistant = messages[1];
expect(assistant.metricSourceJson).toBeTruthy();
const persisted = JSON.parse(assistant.metricSourceJson ?? "{}") as {
keyValues?: Array<{ label: string; value: string; unit?: string }>;
};
expect(persisted.keyValues?.length).toBe(2);
expect(persisted.keyValues?.[0]).toMatchObject({
label: "avg7 systolic",
value: "138",
unit: "mmHg",
window: "last7days",
});
});
it("falls through gracefully when the provider returns prose-only (no sentinel)", async () => {
// Mirrors the mock-client.ts contract — older test doubles and
// refusal fallbacks emit plain prose without the v1.4.22 sentinel.
// The route should not flag the reply malformed; the persisted
// envelope keeps the existing windows/metrics/counts without a
// `keyValues` field.
await seedUserWithSession();
runProviderMock.mockResolvedValue({
result: {
content:
"Compliance has been steady this month — that consistency is doing a lot of quiet work for you. How does that match your own sense of the month?",
tokensUsed: 128,
model: "mock",
providerType: "openai",
},
workingProvider: { providerType: "openai", instance: {} },
fallbackHops: [],
});
const { POST } = await import("@/app/api/insights/chat/route");
const res = await (POST as (req: Request) => Promise<Response>)(
new Request("http://localhost/api/insights/chat", {
method: "POST",
headers: { "content-type": "application/json" },
body: JSON.stringify({ message: "How am I doing this month?" }),
}),
);
expect(res.status).toBe(200);
const text = await readStream(res);
expect(text).toMatch(/"type":"done"/);
const prisma = getPrismaClient();
const messages = await prisma.coachMessage.findMany({
orderBy: { createdAt: "asc" },
});
const assistant = messages.find((m) => m.role === "assistant");
const persisted = JSON.parse(assistant?.metricSourceJson ?? "{}") as {
keyValues?: unknown;
};
expect(persisted.keyValues).toBeUndefined();
});
});