diff --git a/src/lib/insights/__tests__/status-shared.test.ts b/src/lib/insights/__tests__/status-shared.test.ts index 6b3ce1f84..1a0e9f309 100644 --- a/src/lib/insights/__tests__/status-shared.test.ts +++ b/src/lib/insights/__tests__/status-shared.test.ts @@ -1,9 +1,20 @@ import { describe, expect, it } from "vitest"; import { + extractAssessmentSummary, parseSummaryFromContent, stripJsonFences, } from "@/lib/insights/status-shared"; +/** + * The exact malformed shape reported against the resting-pulse Assessment + * (HealthLog#608): a truncated `{ "summary": "…` — an OPEN brace, the property + * name, quotes and an unclosed string, with no closing `}`. Before the fix this + * fell through the parser and rendered raw (braces, `"summary"`, quotes, literal + * `\n`) in the Assessment card. It must resolve to the controlled fallback. + */ +const TRUNCATED_PULSE_ENVELOPE = + '{ "summary": "Your resting pulse has been running lower than the last check'; + /** * The Anthropic + local providers have no native JSON mode, so a compliant * model still routinely wraps its `{ "summary": … }` reply in a ```json @@ -59,4 +70,69 @@ describe("parseSummaryFromContent", () => { "just prose, no json", ); }); + + it("returns empty string for a truncated envelope (never the raw JSON)", () => { + const out = parseSummaryFromContent(TRUNCATED_PULSE_ENVELOPE); + expect(out).toBe(""); + expect(out).not.toContain('"summary"'); + expect(out).not.toContain("{"); + }); +}); + +describe("extractAssessmentSummary", () => { + it("classifies a structured object with a valid summary", () => { + const out = extractAssessmentSummary('{"summary":"hello"}'); + expect(out).toEqual({ kind: "summary", text: "hello" }); + }); + + it("parses a JSON STRING that carries a top-level summary", () => { + // A provider handed back the envelope as a serialized string. It parses to + // the same summary as an inline object — not treated as prose. + const jsonString = JSON.stringify({ + summary: "resting pulse steady at 58", + }); + const out = extractAssessmentSummary(jsonString); + expect(out).toEqual({ + kind: "summary", + text: "resting pulse steady at 58", + }); + }); + + it("classifies a fenced envelope as summary (no native JSON mode)", () => { + const out = extractAssessmentSummary('```json\n{"summary":"hello"}\n```'); + expect(out).toEqual({ kind: "summary", text: "hello" }); + }); + + it("preserves a plain-text reply as prose, verbatim", () => { + const prose = + "Your resting pulse is averaging 58 bpm, in line with baseline."; + expect(extractAssessmentSummary(prose)).toEqual({ + kind: "prose", + text: prose, + }); + }); + + it("does not treat a plain sentence containing a brace as JSON", () => { + const prose = "Your resting pulse sits around 58 bpm {your usual range}."; + expect(extractAssessmentSummary(prose)).toEqual({ + kind: "prose", + text: prose, + }); + }); + + it("resolves a truncated envelope to unparseable (regression: #608)", () => { + expect(extractAssessmentSummary(TRUNCATED_PULSE_ENVELOPE)).toEqual({ + kind: "unparseable", + }); + }); + + it("resolves a summary-less object to unparseable", () => { + expect(extractAssessmentSummary("{}")).toEqual({ kind: "unparseable" }); + }); + + it("resolves an empty-summary object to unparseable", () => { + expect(extractAssessmentSummary('{"summary":""}')).toEqual({ + kind: "unparseable", + }); + }); }); diff --git a/src/lib/insights/__tests__/status-summary-screen.test.ts b/src/lib/insights/__tests__/status-summary-screen.test.ts index d466bf259..c7c669f64 100644 --- a/src/lib/insights/__tests__/status-summary-screen.test.ts +++ b/src/lib/insights/__tests__/status-summary-screen.test.ts @@ -92,3 +92,37 @@ describe("finalizeStatusSummary — clean assessments pass untouched", () => { expect(out.ok).toBe(true); }); }); + +/** + * HealthLog#608 — a truncated `{ "summary": "…` (unclosed, no `}`) used to fall + * through the parser and render raw (braces, `"summary"`, quotes, literal `\n`) + * in the resting-pulse Assessment card. It must now resolve to the withhold + * state so the caller serves its deterministic stub, never the raw envelope. + */ +describe("finalizeStatusSummary — malformed structured output is withheld", () => { + const TRUNCATED = + '{ "summary": "Your resting pulse has been running lower than the last check'; + + it("withholds a truncated envelope instead of surfacing raw JSON", () => { + const out = finalizeStatusSummary(TRUNCATED, "en"); + expect(out.ok).toBe(false); + if (!out.ok) expect(out.reason).toBe("unparseable_output"); + // Nothing about the raw envelope can leak on the ok:false branch. + if (out.ok) { + expect(out.text).not.toContain('"summary"'); + expect(out.text).not.toContain("{"); + } + }); + + it("withholds a summary-less object", () => { + const out = finalizeStatusSummary("{}", "en"); + expect(out.ok).toBe(false); + if (!out.ok) expect(out.reason).toBe("unparseable_output"); + }); + + it("still renders a valid object summary", () => { + const out = finalizeStatusSummary('{"summary":"Steady at 58 bpm."}', "en"); + expect(out.ok).toBe(true); + if (out.ok) expect(out.text).toBe("Steady at 58 bpm."); + }); +}); diff --git a/src/lib/insights/derived/derived-assessment-ai.ts b/src/lib/insights/derived/derived-assessment-ai.ts index 3b142ef60..f17eba298 100644 --- a/src/lib/insights/derived/derived-assessment-ai.ts +++ b/src/lib/insights/derived/derived-assessment-ai.ts @@ -31,9 +31,9 @@ import { withOutputLanguage, } from "@/lib/ai/prompts/output-language"; import { + extractAssessmentSummary, normalizeLocale, normalizeSummaryText, - parseSummaryFromContent, persistStatusInsight, } from "@/lib/insights/status-shared"; import { runStatusCompletion } from "@/lib/insights/status-provider"; @@ -311,7 +311,19 @@ export async function generateDerivedScoreAssessment(args: { return; } - const text = normalizeSummaryText(parseSummaryFromContent(outcome.content)); + // A malformed / truncated / summary-less structured envelope must never + // persist as the derived-score assessment (the raw-JSON class of bug). On an + // unparseable envelope persist nothing — the route keeps serving the + // always-grounded deterministic fallback, so the field stays honest. + const extracted = extractAssessmentSummary(outcome.content); + if (extracted.kind === "unparseable") { + annotate({ + action: { name: "insights.derived-assessment.unparseable" }, + meta: { metric: args.metric }, + }); + return; + } + const text = normalizeSummaryText(extracted.text); if (!text) return; // v1.22 (W6) — score number-grounding gate. The only numbers the prose may diff --git a/src/lib/insights/medication-compliance-status.ts b/src/lib/insights/medication-compliance-status.ts index 1b5b4f9a1..5fc2ba088 100644 --- a/src/lib/insights/medication-compliance-status.ts +++ b/src/lib/insights/medication-compliance-status.ts @@ -30,9 +30,9 @@ import { buildMetricSignal } from "@/lib/insights/metric-signal"; import { degradeStatusSnapshotToBudget } from "@/lib/insights/graded-series"; import { type SupportedLocale, + extractAssessmentSummary, normalizeLocale, normalizeSummaryText, - parseSummaryFromContent, round, } from "@/lib/insights/status-shared"; import { @@ -577,9 +577,32 @@ export async function prepareMedicationComplianceStatusForUser( // does not emit per-medication text. The per-medication cards carry a // placeholder so the UI surfaces a row per active medication; the // overall `summary` is the model-authored assessment. - const summary = normalizeSummaryText( - parseSummaryFromContent(outcome.content), - ); + // A malformed / truncated / summary-less structured envelope must never + // persist as the day's assessment (the raw-JSON class of bug). WITHHOLD: + // serve the deterministic no-key line, write a short negative stub, and + // persist no model text. The per-medication rows still render. + const extracted = extractAssessmentSummary(outcome.content); + if (extracted.kind === "unparseable") { + annotate({ + action: { name: "insights.status.outbound_blocked" }, + meta: { cacheAction, reason: "unparseable_output" }, + }); + returnTimeoutFallback({ + cacheAction, + reason: "screened", + userId, + todayKey, + stubText: getNoKeyMedicationComplianceStatusText(locale), + }); + return { + hasProvider: true, + summary: getNoKeyMedicationComplianceStatusText(locale), + medications: medicationSummaries, + cached: true, + updatedAt: null, + }; + } + const summary = normalizeSummaryText(extracted.text); if (!summary) { throw new Error( "Medication-compliance-status summary was empty after normalization", diff --git a/src/lib/insights/status-shared.ts b/src/lib/insights/status-shared.ts index 454ead431..3bda3ab05 100644 --- a/src/lib/insights/status-shared.ts +++ b/src/lib/insights/status-shared.ts @@ -145,30 +145,120 @@ export function stripJsonFences(content: string): string { } /** - * Pull the `summary` string out of a model completion. The status - * prompts return a `{ "summary": "…" }` envelope; a model that ignores - * the contract and returns bare prose falls back to the raw content. + * Outcome of reading a status completion for its user-facing summary. + * + * - `summary` — a `{ "summary": "…" }` envelope parsed to a non-empty + * string. Render it. + * - `prose` — the model ignored the JSON contract and returned a plain + * sentence. Render it verbatim (back-compat with non-JSON-mode providers). + * - `unparseable` — the model ATTEMPTED the structured envelope but it came + * back malformed, truncated, or summary-less. There is NO safe text to + * render: the caller must fall back to its deterministic stub. This is the + * state that stops a partial `{ "summary": "…` (braces, property name, + * quotes, literal `\n`) from ever reaching the card — the class of bug + * reported against the resting-pulse Assessment. + */ +export type SummaryExtraction = + | { kind: "summary"; text: string } + | { kind: "prose"; text: string } + | { kind: "unparseable" }; + +/** + * Parse a candidate string as a `{ summary }` object. Reports whether the + * string was valid JSON *object* at all (`parsed`) separately from whether it + * carried a usable non-empty `summary` — so the caller can tell a malformed / + * truncated envelope apart from a well-formed object that simply lacks the + * field. + */ +function parseSummaryObject(candidate: string): { + parsed: boolean; + summary: string | null; +} { + try { + const obj = JSON.parse(candidate) as unknown; + if (obj !== null && typeof obj === "object" && !Array.isArray(obj)) { + const value = (obj as { summary?: unknown }).summary; + if (typeof value === "string" && value.trim().length > 0) { + return { parsed: true, summary: value }; + } + // Well-formed object, but no usable summary (e.g. `{}` or `{summary:""}`). + return { parsed: true, summary: null }; + } + // Valid JSON, but not a summary object (array / number / bare string). + return { parsed: false, summary: null }; + } catch { + return { parsed: false, summary: null }; + } +} + +/** + * Did the completion OPEN like a JSON object? A genuine prose sentence never + * starts with `{` (a stray brace mid-sentence does not count), so a `{`-led + * payload that failed both parse attempts above is a malformed / truncated + * envelope — never surfaced raw. Strips a leading code fence first so a + * fenced-but-truncated object is caught too. + */ +function looksLikeJsonObjectAttempt(content: string): boolean { + const afterFence = content.replace(/^```(?:json)?\s*/i, "").trimStart(); + return afterFence.startsWith("{"); +} + +/** + * Read a model completion for its user-facing summary and classify the result. * * Tries the content as-is first (the common, fence-free case), then a - * fence-stripped retry for providers without a native JSON mode. + * fence-stripped retry for providers without a native JSON mode. A completion + * that attempted the `{ summary }` envelope but came back malformed / truncated + * / summary-less resolves to `unparseable` rather than falling through as raw + * text — the caller then serves its deterministic fallback. Only a completion + * that never attempted the envelope (genuine prose) passes through verbatim. + */ +export function extractAssessmentSummary(content: string): SummaryExtraction { + const trimmed = content.trim(); + + // Attempt 1 — the content is itself a JSON object. + const direct = parseSummaryObject(trimmed); + if (direct.parsed) { + return direct.summary + ? { kind: "summary", text: direct.summary } + : { kind: "unparseable" }; + } + + // Attempt 2 — strip a code fence / surrounding prose and retry. + const stripped = stripJsonFences(trimmed); + if (stripped !== trimmed) { + const viaFence = parseSummaryObject(stripped); + if (viaFence.parsed) { + return viaFence.summary + ? { kind: "summary", text: viaFence.summary } + : { kind: "unparseable" }; + } + } + + // Neither parse produced a summary object. If the payload OPENED like an + // object, the model attempted the envelope and it came back malformed or + // truncated — resolve to the controlled fallback, never the raw braces / + // property name / quotes / literal `\n`. Genuine prose falls through. + if (looksLikeJsonObjectAttempt(trimmed)) { + return { kind: "unparseable" }; + } + return { kind: "prose", text: content }; +} + +/** + * Pull the `summary` string out of a model completion. The status prompts + * return a `{ "summary": "…" }` envelope; a model that ignores the contract and + * returns bare prose falls back to the raw content. A malformed / truncated / + * summary-less structured envelope resolves to an EMPTY string (never the raw + * JSON) so no caller can surface partial structured output. + * + * Prefer `extractAssessmentSummary` in new code: it distinguishes an + * unparseable envelope from genuinely empty prose, which this thin wrapper + * collapses to `""`. */ export function parseSummaryFromContent(content: string): string { - try { - const parsed = JSON.parse(content) as { summary?: string }; - if (typeof parsed.summary === "string") return parsed.summary; - } catch { - // not directly parseable — try a fence-stripped retry below - } - try { - const stripped = stripJsonFences(content); - if (stripped !== content) { - const parsed = JSON.parse(stripped) as { summary?: string }; - if (typeof parsed.summary === "string") return parsed.summary; - } - } catch { - // still not JSON — fall through to the raw content - } - return content; + const extracted = extractAssessmentSummary(content); + return extracted.kind === "unparseable" ? "" : extracted.text; } /** @@ -230,7 +320,8 @@ export async function persistStatusInsight(args: { * withholds. */ export type StatusSummaryOutcome = - { ok: true; text: string } | { ok: false; reason: OutboundReason }; + | { ok: true; text: string } + | { ok: false; reason: OutboundReason | "unparseable_output" }; /** * The single finalize step every per-metric / biomarker / derived-score status @@ -263,7 +354,14 @@ export function finalizeStatusSummary( content: string, locale: SupportedLocale, ): StatusSummaryOutcome { - const text = normalizeSummaryText(parseSummaryFromContent(content)); + const extracted = extractAssessmentSummary(content); + // A malformed / truncated / summary-less structured envelope has no safe + // text to render. WITHHOLD — the caller serves its deterministic stub. Only + // the `unparseable` discriminator is logged, never the assessment content. + if (extracted.kind === "unparseable") { + return { ok: false, reason: "unparseable_output" }; + } + const text = normalizeSummaryText(extracted.text); if (!text) return { ok: true, text: "" }; const decision = screenModelOutput(text, locale, INSIGHTS_CONTRACTS); if (decision.block && decision.reason) {