fix(insights): withhold malformed status output instead of rendering raw JSON

A truncated or malformed { "summary": "..." } envelope fell through the
status-card parser and rendered raw — braces, the summary property name,
quotes, and literal \n — in the Assessment card (resting pulse and the
other per-metric cards). parseSummaryFromContent returned the raw content
on any JSON parse failure, and a truncated envelope with no closing brace
also fell into that path, so partial structured output reached the reader.

Split the read into extractAssessmentSummary, which classifies a completion
as a parsed summary, genuine prose, or an unparseable structured envelope.
A payload that opened like a JSON object but failed to parse (or parsed
without a usable summary) resolves to the controlled fallback rather than
raw text; plain prose that merely contains a brace is still preserved.
finalizeStatusSummary withholds an unparseable envelope so every status
card serves its deterministic stub, and the two direct callers
(medication compliance, derived-score assessment) get the same guard.

Refs #608
This commit is contained in:
Marc-André Bombeck committed 2026-07-24 07:45:02 +02:00
1 parent 4f95afa33c
commit 93c0d616bb
5 files changed
+271 -28

No files matched your search

@@ -1,9 +1,20 @@
import { describe, expect, it } from "vitest";
import {
extractAssessmentSummary,
parseSummaryFromContent,
stripJsonFences,
} from "@/lib/insights/status-shared";
/**
* The exact malformed shape reported against the resting-pulse Assessment
* (HealthLog#608): a truncated `{ "summary": "…` — an OPEN brace, the property
* name, quotes and an unclosed string, with no closing `}`. Before the fix this
* fell through the parser and rendered raw (braces, `"summary"`, quotes, literal
* `\n`) in the Assessment card. It must resolve to the controlled fallback.
*/
const TRUNCATED_PULSE_ENVELOPE =
'{ "summary": "Your resting pulse has been running lower than the last check';
/**
* The Anthropic + local providers have no native JSON mode, so a compliant
* model still routinely wraps its `{ "summary": … }` reply in a ```json
@@ -59,4 +70,69 @@ describe("parseSummaryFromContent", () => {
"just prose, no json",
);
});
it("returns empty string for a truncated envelope (never the raw JSON)", () => {
const out = parseSummaryFromContent(TRUNCATED_PULSE_ENVELOPE);
expect(out).toBe("");
expect(out).not.toContain('"summary"');
expect(out).not.toContain("{");
});
});
describe("extractAssessmentSummary", () => {
it("classifies a structured object with a valid summary", () => {
const out = extractAssessmentSummary('{"summary":"hello"}');
expect(out).toEqual({ kind: "summary", text: "hello" });
});
it("parses a JSON STRING that carries a top-level summary", () => {
// A provider handed back the envelope as a serialized string. It parses to
// the same summary as an inline object — not treated as prose.
const jsonString = JSON.stringify({
summary: "resting pulse steady at 58",
});
const out = extractAssessmentSummary(jsonString);
expect(out).toEqual({
kind: "summary",
text: "resting pulse steady at 58",
});
});
it("classifies a fenced envelope as summary (no native JSON mode)", () => {
const out = extractAssessmentSummary('```json\n{"summary":"hello"}\n```');
expect(out).toEqual({ kind: "summary", text: "hello" });
});
it("preserves a plain-text reply as prose, verbatim", () => {
const prose =
"Your resting pulse is averaging 58 bpm, in line with baseline.";
expect(extractAssessmentSummary(prose)).toEqual({
kind: "prose",
text: prose,
});
});
it("does not treat a plain sentence containing a brace as JSON", () => {
const prose = "Your resting pulse sits around 58 bpm {your usual range}.";
expect(extractAssessmentSummary(prose)).toEqual({
kind: "prose",
text: prose,
});
});
it("resolves a truncated envelope to unparseable (regression: #608)", () => {
expect(extractAssessmentSummary(TRUNCATED_PULSE_ENVELOPE)).toEqual({
kind: "unparseable",
});
});
it("resolves a summary-less object to unparseable", () => {
expect(extractAssessmentSummary("{}")).toEqual({ kind: "unparseable" });
});
it("resolves an empty-summary object to unparseable", () => {
expect(extractAssessmentSummary('{"summary":""}')).toEqual({
kind: "unparseable",
});
});
});
@@ -92,3 +92,37 @@ describe("finalizeStatusSummary — clean assessments pass untouched", () => {
expect(out.ok).toBe(true);
});
});
/**
* HealthLog#608 — a truncated `{ "summary": "…` (unclosed, no `}`) used to fall
* through the parser and render raw (braces, `"summary"`, quotes, literal `\n`)
* in the resting-pulse Assessment card. It must now resolve to the withhold
* state so the caller serves its deterministic stub, never the raw envelope.
*/
describe("finalizeStatusSummary — malformed structured output is withheld", () => {
const TRUNCATED =
'{ "summary": "Your resting pulse has been running lower than the last check';
it("withholds a truncated envelope instead of surfacing raw JSON", () => {
const out = finalizeStatusSummary(TRUNCATED, "en");
expect(out.ok).toBe(false);
if (!out.ok) expect(out.reason).toBe("unparseable_output");
// Nothing about the raw envelope can leak on the ok:false branch.
if (out.ok) {
expect(out.text).not.toContain('"summary"');
expect(out.text).not.toContain("{");
}
});
it("withholds a summary-less object", () => {
const out = finalizeStatusSummary("{}", "en");
expect(out.ok).toBe(false);
if (!out.ok) expect(out.reason).toBe("unparseable_output");
});
it("still renders a valid object summary", () => {
const out = finalizeStatusSummary('{"summary":"Steady at 58 bpm."}', "en");
expect(out.ok).toBe(true);
if (out.ok) expect(out.text).toBe("Steady at 58 bpm.");
});
});
@@ -31,9 +31,9 @@ import {
withOutputLanguage,
} from "@/lib/ai/prompts/output-language";
import {
extractAssessmentSummary,
normalizeLocale,
normalizeSummaryText,
parseSummaryFromContent,
persistStatusInsight,
} from "@/lib/insights/status-shared";
import { runStatusCompletion } from "@/lib/insights/status-provider";
@@ -311,7 +311,19 @@ export async function generateDerivedScoreAssessment(args: {
return;
}
const text = normalizeSummaryText(parseSummaryFromContent(outcome.content));
// A malformed / truncated / summary-less structured envelope must never
// persist as the derived-score assessment (the raw-JSON class of bug). On an
// unparseable envelope persist nothing — the route keeps serving the
// always-grounded deterministic fallback, so the field stays honest.
const extracted = extractAssessmentSummary(outcome.content);
if (extracted.kind === "unparseable") {
annotate({
action: { name: "insights.derived-assessment.unparseable" },
meta: { metric: args.metric },
});
return;
}
const text = normalizeSummaryText(extracted.text);
if (!text) return;
// v1.22 (W6) — score number-grounding gate. The only numbers the prose may
@@ -30,9 +30,9 @@ import { buildMetricSignal } from "@/lib/insights/metric-signal";
import { degradeStatusSnapshotToBudget } from "@/lib/insights/graded-series";
import {
type SupportedLocale,
extractAssessmentSummary,
normalizeLocale,
normalizeSummaryText,
parseSummaryFromContent,
round,
} from "@/lib/insights/status-shared";
import {
@@ -577,9 +577,32 @@ export async function prepareMedicationComplianceStatusForUser(
// does not emit per-medication text. The per-medication cards carry a
// placeholder so the UI surfaces a row per active medication; the
// overall `summary` is the model-authored assessment.
const summary = normalizeSummaryText(
parseSummaryFromContent(outcome.content),
);
// A malformed / truncated / summary-less structured envelope must never
// persist as the day's assessment (the raw-JSON class of bug). WITHHOLD:
// serve the deterministic no-key line, write a short negative stub, and
// persist no model text. The per-medication rows still render.
const extracted = extractAssessmentSummary(outcome.content);
if (extracted.kind === "unparseable") {
annotate({
action: { name: "insights.status.outbound_blocked" },
meta: { cacheAction, reason: "unparseable_output" },
});
returnTimeoutFallback({
cacheAction,
reason: "screened",
userId,
todayKey,
stubText: getNoKeyMedicationComplianceStatusText(locale),
});
return {
hasProvider: true,
summary: getNoKeyMedicationComplianceStatusText(locale),
medications: medicationSummaries,
cached: true,
updatedAt: null,
};
}
const summary = normalizeSummaryText(extracted.text);
if (!summary) {
throw new Error(
"Medication-compliance-status summary was empty after normalization",
+120 -22
View File
@@ -145,30 +145,120 @@ export function stripJsonFences(content: string): string {
}
/**
* Pull the `summary` string out of a model completion. The status
* prompts return a `{ "summary": "…" }` envelope; a model that ignores
* the contract and returns bare prose falls back to the raw content.
* Outcome of reading a status completion for its user-facing summary.
*
* - `summary` — a `{ "summary": "…" }` envelope parsed to a non-empty
* string. Render it.
* - `prose` — the model ignored the JSON contract and returned a plain
* sentence. Render it verbatim (back-compat with non-JSON-mode providers).
* - `unparseable` — the model ATTEMPTED the structured envelope but it came
* back malformed, truncated, or summary-less. There is NO safe text to
* render: the caller must fall back to its deterministic stub. This is the
* state that stops a partial `{ "summary": "…` (braces, property name,
* quotes, literal `\n`) from ever reaching the card — the class of bug
* reported against the resting-pulse Assessment.
*/
export type SummaryExtraction =
| { kind: "summary"; text: string }
| { kind: "prose"; text: string }
| { kind: "unparseable" };
/**
* Parse a candidate string as a `{ summary }` object. Reports whether the
* string was valid JSON *object* at all (`parsed`) separately from whether it
* carried a usable non-empty `summary` — so the caller can tell a malformed /
* truncated envelope apart from a well-formed object that simply lacks the
* field.
*/
function parseSummaryObject(candidate: string): {
parsed: boolean;
summary: string | null;
} {
try {
const obj = JSON.parse(candidate) as unknown;
if (obj !== null && typeof obj === "object" && !Array.isArray(obj)) {
const value = (obj as { summary?: unknown }).summary;
if (typeof value === "string" && value.trim().length > 0) {
return { parsed: true, summary: value };
}
// Well-formed object, but no usable summary (e.g. `{}` or `{summary:""}`).
return { parsed: true, summary: null };
}
// Valid JSON, but not a summary object (array / number / bare string).
return { parsed: false, summary: null };
} catch {
return { parsed: false, summary: null };
}
}
/**
* Did the completion OPEN like a JSON object? A genuine prose sentence never
* starts with `{` (a stray brace mid-sentence does not count), so a `{`-led
* payload that failed both parse attempts above is a malformed / truncated
* envelope — never surfaced raw. Strips a leading code fence first so a
* fenced-but-truncated object is caught too.
*/
function looksLikeJsonObjectAttempt(content: string): boolean {
const afterFence = content.replace(/^```(?:json)?\s*/i, "").trimStart();
return afterFence.startsWith("{");
}
/**
* Read a model completion for its user-facing summary and classify the result.
*
* Tries the content as-is first (the common, fence-free case), then a
* fence-stripped retry for providers without a native JSON mode.
* fence-stripped retry for providers without a native JSON mode. A completion
* that attempted the `{ summary }` envelope but came back malformed / truncated
* / summary-less resolves to `unparseable` rather than falling through as raw
* text — the caller then serves its deterministic fallback. Only a completion
* that never attempted the envelope (genuine prose) passes through verbatim.
*/
export function extractAssessmentSummary(content: string): SummaryExtraction {
const trimmed = content.trim();
// Attempt 1 — the content is itself a JSON object.
const direct = parseSummaryObject(trimmed);
if (direct.parsed) {
return direct.summary
? { kind: "summary", text: direct.summary }
: { kind: "unparseable" };
}
// Attempt 2 — strip a code fence / surrounding prose and retry.
const stripped = stripJsonFences(trimmed);
if (stripped !== trimmed) {
const viaFence = parseSummaryObject(stripped);
if (viaFence.parsed) {
return viaFence.summary
? { kind: "summary", text: viaFence.summary }
: { kind: "unparseable" };
}
}
// Neither parse produced a summary object. If the payload OPENED like an
// object, the model attempted the envelope and it came back malformed or
// truncated — resolve to the controlled fallback, never the raw braces /
// property name / quotes / literal `\n`. Genuine prose falls through.
if (looksLikeJsonObjectAttempt(trimmed)) {
return { kind: "unparseable" };
}
return { kind: "prose", text: content };
}
/**
* Pull the `summary` string out of a model completion. The status prompts
* return a `{ "summary": "…" }` envelope; a model that ignores the contract and
* returns bare prose falls back to the raw content. A malformed / truncated /
* summary-less structured envelope resolves to an EMPTY string (never the raw
* JSON) so no caller can surface partial structured output.
*
* Prefer `extractAssessmentSummary` in new code: it distinguishes an
* unparseable envelope from genuinely empty prose, which this thin wrapper
* collapses to `""`.
*/
export function parseSummaryFromContent(content: string): string {
try {
const parsed = JSON.parse(content) as { summary?: string };
if (typeof parsed.summary === "string") return parsed.summary;
} catch {
// not directly parseable — try a fence-stripped retry below
}
try {
const stripped = stripJsonFences(content);
if (stripped !== content) {
const parsed = JSON.parse(stripped) as { summary?: string };
if (typeof parsed.summary === "string") return parsed.summary;
}
} catch {
// still not JSON — fall through to the raw content
}
return content;
const extracted = extractAssessmentSummary(content);
return extracted.kind === "unparseable" ? "" : extracted.text;
}
/**
@@ -230,7 +320,8 @@ export async function persistStatusInsight(args: {
* withholds.
*/
export type StatusSummaryOutcome =
{ ok: true; text: string } | { ok: false; reason: OutboundReason };
| { ok: true; text: string }
| { ok: false; reason: OutboundReason | "unparseable_output" };
/**
* The single finalize step every per-metric / biomarker / derived-score status
@@ -263,7 +354,14 @@ export function finalizeStatusSummary(
content: string,
locale: SupportedLocale,
): StatusSummaryOutcome {
const text = normalizeSummaryText(parseSummaryFromContent(content));
const extracted = extractAssessmentSummary(content);
// A malformed / truncated / summary-less structured envelope has no safe
// text to render. WITHHOLD — the caller serves its deterministic stub. Only
// the `unparseable` discriminator is logged, never the assessment content.
if (extracted.kind === "unparseable") {
return { ok: false, reason: "unparseable_output" };
}
const text = normalizeSummaryText(extracted.text);
if (!text) return { ok: true, text: "" };
const decision = screenModelOutput(text, locale, INSIGHTS_CONTRACTS);
if (decision.block && decision.reason) {