diff --git a/src/app/api/insights/chat/route.ts b/src/app/api/insights/chat/route.ts index e0c5d1693..fde99d741 100644 --- a/src/app/api/insights/chat/route.ts +++ b/src/app/api/insights/chat/route.ts @@ -300,11 +300,24 @@ Reply now as the assistant, in ${locale === "de" ? "German" : "English"}.`; if (sentinel.malformed) { // Graceful degrade: log so ops can spot a provider whose // sentinel format has drifted, but pass the prose through - // unchanged. + // unchanged. v1.4.23 H1 — split the annotation: + // - parse_partial: at least one row parsed AND at least one + // row failed (mixed-format drift on a single reply) + // - parse_failed: the whole block was unusable + // Both annotations carry the per-line `reasons` array so an ops + // dashboard can attribute the failure cause without re-running + // the parser. + const reasons = sentinel.malformedEntries.map((entry) => entry.reason); + const annotationName = + sentinel.keyValues.length > 0 && sentinel.malformedEntries.length > 0 + ? "coach.keyvalues.parse_partial" + : "coach.keyvalues.parse_failed"; annotate({ - action: { name: "coach.keyvalues.parse_failed" }, + action: { name: annotationName }, meta: { kept: sentinel.keyValues.length, + malformedCount: sentinel.malformedEntries.length, + reasons, promptVersion: PROMPT_VERSION, }, }); diff --git a/src/lib/ai/coach/__tests__/keyvalues-parse.test.ts b/src/lib/ai/coach/__tests__/keyvalues-parse.test.ts index 41f27f7a5..6b1eee0bd 100644 --- a/src/lib/ai/coach/__tests__/keyvalues-parse.test.ts +++ b/src/lib/ai/coach/__tests__/keyvalues-parse.test.ts @@ -1,6 +1,10 @@ import { describe, expect, it } from "vitest"; -import { parseKeyValueLine, parseKeyValuesSentinel } from "../keyvalues"; +import { + parseKeyValueLine, + parseKeyValuesSentinel, + tryParseKeyValueLine, +} from "../keyvalues"; /** * v1.4.22 Wave 3 B1+B2 — Coach evidence-block sentinel parser. @@ -91,6 +95,7 @@ describe("parseKeyValuesSentinel", () => { }, ]); expect(out.malformed).toBe(false); + expect(out.malformedEntries).toEqual([]); }); it("returns prose untouched when no sentinel is present", () => { @@ -100,6 +105,7 @@ describe("parseKeyValuesSentinel", () => { expect(out.prose).toBe("Just a qualitative reply with no numbers cited."); expect(out.keyValues).toEqual([]); expect(out.malformed).toBe(false); + expect(out.malformedEntries).toEqual([]); }); it("graceful-degrades when the closing ---END--- is missing", () => { @@ -115,6 +121,9 @@ describe("parseKeyValuesSentinel", () => { expect(out.prose).toBe("Prose still streams cleanly."); expect(out.keyValues.length).toBe(1); expect(out.malformed).toBe(true); + expect(out.malformedEntries).toEqual([ + { rawLine: "---END---", reason: "no_END_marker" }, + ]); }); it("clips the block to 8 entries (prompt-contract cap)", () => { @@ -143,7 +152,7 @@ describe("parseKeyValuesSentinel", () => { expect(out.malformed).toBe(true); }); - it("silently drops malformed lines mixed in with valid ones", () => { + it("records malformed lines mixed in with valid ones (v1.4.23 H1)", () => { const raw = [ "Prose.", "", @@ -160,6 +169,65 @@ describe("parseKeyValuesSentinel", () => { "avg30 systolic", "avg30 mood", ]); + // Partial malformed surfaces — the good rows survive while the + // bad ones land in the entry array with typed reasons. + expect(out.malformed).toBe(true); + expect(out.malformedEntries).toEqual([ + { rawLine: "not a valid line", reason: "missing_colon" }, + { rawLine: ":no-label-here", reason: "missing_colon" }, + ]); + }); + + it("flags label_overflow when a row's label exceeds the cap", () => { + const longLabel = "x".repeat(120); + const raw = [ + "---KEYVALUES---", + "ok label: 138 [mmHg] (last30days)", + `${longLabel}: 99 [mmHg]`, + "---END---", + ].join("\n"); + const out = parseKeyValuesSentinel(raw); + expect(out.keyValues.length).toBe(1); + expect(out.keyValues[0].label).toBe("ok label"); + expect(out.malformed).toBe(true); + expect(out.malformedEntries).toEqual([ + { rawLine: `${longLabel}: 99 [mmHg]`, reason: "label_overflow" }, + ]); + }); + + it("flags value_overflow when a row's value exceeds the cap", () => { + const longValue = "9".repeat(220); + const raw = [ + "---KEYVALUES---", + `avg30 systolic: ${longValue} [mmHg]`, + "ok label: 138 [mmHg] (last30days)", + "---END---", + ].join("\n"); + const out = parseKeyValuesSentinel(raw); + expect(out.keyValues.length).toBe(1); + expect(out.keyValues[0].label).toBe("ok label"); + expect(out.malformed).toBe(true); + expect(out.malformedEntries[0]?.reason).toBe("value_overflow"); + }); + + it("flags byte_overflow when the block exceeds the 1 KB cap", () => { + const padded = Array.from( + { length: 60 }, + (_, i) => `label-${i}: ${i.toString().padEnd(15, "x")}`, + ); + const raw = ["---KEYVALUES---", ...padded, "---END---"].join("\n"); + const out = parseKeyValuesSentinel(raw); + expect(out.malformed).toBe(true); + expect( + out.malformedEntries.some((entry) => entry.reason === "byte_overflow"), + ).toBe(true); + }); + + it("tryParseKeyValueLine surfaces missing_colon for separator-less rows", () => { + expect(tryParseKeyValueLine("no separator here")).toEqual({ + ok: false, + reason: "missing_colon", + }); }); it("returns malformed when the sentinel block is empty", () => { @@ -168,6 +236,8 @@ describe("parseKeyValuesSentinel", () => { expect(out.keyValues).toEqual([]); expect(out.malformed).toBe(true); expect(out.prose).toBe("Prose."); + // No body lines to record — only the block-level state flips. + expect(out.malformedEntries).toEqual([]); }); it("returns prose and an empty array for empty input", () => { @@ -175,5 +245,6 @@ describe("parseKeyValuesSentinel", () => { expect(out.prose).toBe(""); expect(out.keyValues).toEqual([]); expect(out.malformed).toBe(false); + expect(out.malformedEntries).toEqual([]); }); }); diff --git a/src/lib/ai/coach/keyvalues.ts b/src/lib/ai/coach/keyvalues.ts index 99cdcf45f..b1d2c1fd2 100644 --- a/src/lib/ai/coach/keyvalues.ts +++ b/src/lib/ai/coach/keyvalues.ts @@ -19,11 +19,14 @@ * - Hard cap 1 KB on the sentinel block (prompt-injection guard). * - Hard cap 8 lines kept (per the prompt contract). * - Each line is validated against `coachKeyValueSchema`; malformed - * lines are dropped silently. + * lines surface in `parsed.malformed[]` with a typed reason + * (v1.4.23 H1 — replaced the silent-drop path). * - When the sentinel pair is malformed (e.g. no closing `---END---`, - * unbalanced sentinels), the parser returns the original prose - * untouched and emits no keyValues — the caller logs a wide-event - * `coach.keyvalues.parse_failed` for ops visibility. + * unbalanced sentinels), the parser still returns whatever rows it + * could parse plus a malformed flag; the caller logs + * `coach.keyvalues.parse_failed` (full block invalid) or + * `coach.keyvalues.parse_partial` (mixed valid + invalid) for ops + * visibility. */ import { coachKeyValueSchema, type CoachKeyValue } from "./types"; @@ -31,31 +34,76 @@ import { coachKeyValueSchema, type CoachKeyValue } from "./types"; const SENTINEL_BYTE_CAP = 1024; /** Max key/value rows kept inside the block (prompt-contract value). */ const SENTINEL_LINE_CAP = 8; +/** Per-line value cap before we reject the row (defence-in-depth). */ +const VALUE_BYTE_CAP = 200; +/** Per-line label cap mirroring the prompt-contract value of 40 chars. */ +const LABEL_BYTE_CAP = 80; const OPEN_SENTINEL = "---KEYVALUES---"; const CLOSE_SENTINEL = "---END---"; +/** + * Why a candidate row failed to parse. Surfaces to the caller via + * `SentinelParseResult.malformed[]` so ops dashboards can attribute + * partial failures (mixed valid + invalid rows) without the silent + * drops the v1.4.22 W3 implementation paid. + * + * - `missing_colon` — line had no `:` separator. + * - `value_overflow` — value exceeded `VALUE_BYTE_CAP`. + * - `label_overflow` — label exceeded `LABEL_BYTE_CAP`. + * - `no_END_marker` — the closing `---END---` was missing. + * - `byte_overflow` — block exceeded `SENTINEL_BYTE_CAP`. + * - `schema_invalid` — row passed shape but Zod rejected it + * (e.g. empty value after stripping). + */ +export type SentinelMalformedReason = + | "missing_colon" + | "value_overflow" + | "label_overflow" + | "no_END_marker" + | "byte_overflow" + | "schema_invalid"; + +export interface SentinelMalformedEntry { + rawLine: string; + reason: SentinelMalformedReason; +} + +/** + * Result of parsing one body line. Either returns the `CoachKeyValue` + * shape or a typed reason the line failed (so the caller can surface + * partial-failure observability in the wide-event annotation). + */ +export type LineParseOutcome = + | { ok: true; value: CoachKeyValue } + | { ok: false; reason: SentinelMalformedReason }; + /** * Parse one body line of the form * `