mirror of
https://github.com/MBombeck/HealthLog.git
synced 2026-11-08 12:32:43 +00:00
fix(insights): withhold malformed status output instead of rendering raw JSON
A truncated or malformed { "summary": "..." } envelope fell through the
status-card parser and rendered raw — braces, the summary property name,
quotes, and literal \n — in the Assessment card (resting pulse and the
other per-metric cards). parseSummaryFromContent returned the raw content
on any JSON parse failure, and a truncated envelope with no closing brace
also fell into that path, so partial structured output reached the reader.
Split the read into extractAssessmentSummary, which classifies a completion
as a parsed summary, genuine prose, or an unparseable structured envelope.
A payload that opened like a JSON object but failed to parse (or parsed
without a usable summary) resolves to the controlled fallback rather than
raw text; plain prose that merely contains a brace is still preserved.
finalizeStatusSummary withholds an unparseable envelope so every status
card serves its deterministic stub, and the two direct callers
(medication compliance, derived-score assessment) get the same guard.
Refs #608
This commit is contained in:
1 parent
4f95afa33c
commit
93c0d616bb
5 files changed
+271
-28
No files matched your search
@@ -1,9 +1,20 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import {
|
||||
extractAssessmentSummary,
|
||||
parseSummaryFromContent,
|
||||
stripJsonFences,
|
||||
} from "@/lib/insights/status-shared";
|
||||
|
||||
/**
|
||||
* The exact malformed shape reported against the resting-pulse Assessment
|
||||
* (HealthLog#608): a truncated `{ "summary": "…` — an OPEN brace, the property
|
||||
* name, quotes and an unclosed string, with no closing `}`. Before the fix this
|
||||
* fell through the parser and rendered raw (braces, `"summary"`, quotes, literal
|
||||
* `\n`) in the Assessment card. It must resolve to the controlled fallback.
|
||||
*/
|
||||
const TRUNCATED_PULSE_ENVELOPE =
|
||||
'{ "summary": "Your resting pulse has been running lower than the last check';
|
||||
|
||||
/**
|
||||
* The Anthropic + local providers have no native JSON mode, so a compliant
|
||||
* model still routinely wraps its `{ "summary": … }` reply in a ```json
|
||||
@@ -59,4 +70,69 @@ describe("parseSummaryFromContent", () => {
|
||||
"just prose, no json",
|
||||
);
|
||||
});
|
||||
|
||||
it("returns empty string for a truncated envelope (never the raw JSON)", () => {
|
||||
const out = parseSummaryFromContent(TRUNCATED_PULSE_ENVELOPE);
|
||||
expect(out).toBe("");
|
||||
expect(out).not.toContain('"summary"');
|
||||
expect(out).not.toContain("{");
|
||||
});
|
||||
});
|
||||
|
||||
describe("extractAssessmentSummary", () => {
|
||||
it("classifies a structured object with a valid summary", () => {
|
||||
const out = extractAssessmentSummary('{"summary":"hello"}');
|
||||
expect(out).toEqual({ kind: "summary", text: "hello" });
|
||||
});
|
||||
|
||||
it("parses a JSON STRING that carries a top-level summary", () => {
|
||||
// A provider handed back the envelope as a serialized string. It parses to
|
||||
// the same summary as an inline object — not treated as prose.
|
||||
const jsonString = JSON.stringify({
|
||||
summary: "resting pulse steady at 58",
|
||||
});
|
||||
const out = extractAssessmentSummary(jsonString);
|
||||
expect(out).toEqual({
|
||||
kind: "summary",
|
||||
text: "resting pulse steady at 58",
|
||||
});
|
||||
});
|
||||
|
||||
it("classifies a fenced envelope as summary (no native JSON mode)", () => {
|
||||
const out = extractAssessmentSummary('```json\n{"summary":"hello"}\n```');
|
||||
expect(out).toEqual({ kind: "summary", text: "hello" });
|
||||
});
|
||||
|
||||
it("preserves a plain-text reply as prose, verbatim", () => {
|
||||
const prose =
|
||||
"Your resting pulse is averaging 58 bpm, in line with baseline.";
|
||||
expect(extractAssessmentSummary(prose)).toEqual({
|
||||
kind: "prose",
|
||||
text: prose,
|
||||
});
|
||||
});
|
||||
|
||||
it("does not treat a plain sentence containing a brace as JSON", () => {
|
||||
const prose = "Your resting pulse sits around 58 bpm {your usual range}.";
|
||||
expect(extractAssessmentSummary(prose)).toEqual({
|
||||
kind: "prose",
|
||||
text: prose,
|
||||
});
|
||||
});
|
||||
|
||||
it("resolves a truncated envelope to unparseable (regression: #608)", () => {
|
||||
expect(extractAssessmentSummary(TRUNCATED_PULSE_ENVELOPE)).toEqual({
|
||||
kind: "unparseable",
|
||||
});
|
||||
});
|
||||
|
||||
it("resolves a summary-less object to unparseable", () => {
|
||||
expect(extractAssessmentSummary("{}")).toEqual({ kind: "unparseable" });
|
||||
});
|
||||
|
||||
it("resolves an empty-summary object to unparseable", () => {
|
||||
expect(extractAssessmentSummary('{"summary":""}')).toEqual({
|
||||
kind: "unparseable",
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -92,3 +92,37 @@ describe("finalizeStatusSummary — clean assessments pass untouched", () => {
|
||||
expect(out.ok).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
/**
|
||||
* HealthLog#608 — a truncated `{ "summary": "…` (unclosed, no `}`) used to fall
|
||||
* through the parser and render raw (braces, `"summary"`, quotes, literal `\n`)
|
||||
* in the resting-pulse Assessment card. It must now resolve to the withhold
|
||||
* state so the caller serves its deterministic stub, never the raw envelope.
|
||||
*/
|
||||
describe("finalizeStatusSummary — malformed structured output is withheld", () => {
|
||||
const TRUNCATED =
|
||||
'{ "summary": "Your resting pulse has been running lower than the last check';
|
||||
|
||||
it("withholds a truncated envelope instead of surfacing raw JSON", () => {
|
||||
const out = finalizeStatusSummary(TRUNCATED, "en");
|
||||
expect(out.ok).toBe(false);
|
||||
if (!out.ok) expect(out.reason).toBe("unparseable_output");
|
||||
// Nothing about the raw envelope can leak on the ok:false branch.
|
||||
if (out.ok) {
|
||||
expect(out.text).not.toContain('"summary"');
|
||||
expect(out.text).not.toContain("{");
|
||||
}
|
||||
});
|
||||
|
||||
it("withholds a summary-less object", () => {
|
||||
const out = finalizeStatusSummary("{}", "en");
|
||||
expect(out.ok).toBe(false);
|
||||
if (!out.ok) expect(out.reason).toBe("unparseable_output");
|
||||
});
|
||||
|
||||
it("still renders a valid object summary", () => {
|
||||
const out = finalizeStatusSummary('{"summary":"Steady at 58 bpm."}', "en");
|
||||
expect(out.ok).toBe(true);
|
||||
if (out.ok) expect(out.text).toBe("Steady at 58 bpm.");
|
||||
});
|
||||
});
|
||||
@@ -31,9 +31,9 @@ import {
|
||||
withOutputLanguage,
|
||||
} from "@/lib/ai/prompts/output-language";
|
||||
import {
|
||||
extractAssessmentSummary,
|
||||
normalizeLocale,
|
||||
normalizeSummaryText,
|
||||
parseSummaryFromContent,
|
||||
persistStatusInsight,
|
||||
} from "@/lib/insights/status-shared";
|
||||
import { runStatusCompletion } from "@/lib/insights/status-provider";
|
||||
@@ -311,7 +311,19 @@ export async function generateDerivedScoreAssessment(args: {
|
||||
return;
|
||||
}
|
||||
|
||||
const text = normalizeSummaryText(parseSummaryFromContent(outcome.content));
|
||||
// A malformed / truncated / summary-less structured envelope must never
|
||||
// persist as the derived-score assessment (the raw-JSON class of bug). On an
|
||||
// unparseable envelope persist nothing — the route keeps serving the
|
||||
// always-grounded deterministic fallback, so the field stays honest.
|
||||
const extracted = extractAssessmentSummary(outcome.content);
|
||||
if (extracted.kind === "unparseable") {
|
||||
annotate({
|
||||
action: { name: "insights.derived-assessment.unparseable" },
|
||||
meta: { metric: args.metric },
|
||||
});
|
||||
return;
|
||||
}
|
||||
const text = normalizeSummaryText(extracted.text);
|
||||
if (!text) return;
|
||||
|
||||
// v1.22 (W6) — score number-grounding gate. The only numbers the prose may
|
||||
|
||||
@@ -30,9 +30,9 @@ import { buildMetricSignal } from "@/lib/insights/metric-signal";
|
||||
import { degradeStatusSnapshotToBudget } from "@/lib/insights/graded-series";
|
||||
import {
|
||||
type SupportedLocale,
|
||||
extractAssessmentSummary,
|
||||
normalizeLocale,
|
||||
normalizeSummaryText,
|
||||
parseSummaryFromContent,
|
||||
round,
|
||||
} from "@/lib/insights/status-shared";
|
||||
import {
|
||||
@@ -577,9 +577,32 @@ export async function prepareMedicationComplianceStatusForUser(
|
||||
// does not emit per-medication text. The per-medication cards carry a
|
||||
// placeholder so the UI surfaces a row per active medication; the
|
||||
// overall `summary` is the model-authored assessment.
|
||||
const summary = normalizeSummaryText(
|
||||
parseSummaryFromContent(outcome.content),
|
||||
);
|
||||
// A malformed / truncated / summary-less structured envelope must never
|
||||
// persist as the day's assessment (the raw-JSON class of bug). WITHHOLD:
|
||||
// serve the deterministic no-key line, write a short negative stub, and
|
||||
// persist no model text. The per-medication rows still render.
|
||||
const extracted = extractAssessmentSummary(outcome.content);
|
||||
if (extracted.kind === "unparseable") {
|
||||
annotate({
|
||||
action: { name: "insights.status.outbound_blocked" },
|
||||
meta: { cacheAction, reason: "unparseable_output" },
|
||||
});
|
||||
returnTimeoutFallback({
|
||||
cacheAction,
|
||||
reason: "screened",
|
||||
userId,
|
||||
todayKey,
|
||||
stubText: getNoKeyMedicationComplianceStatusText(locale),
|
||||
});
|
||||
return {
|
||||
hasProvider: true,
|
||||
summary: getNoKeyMedicationComplianceStatusText(locale),
|
||||
medications: medicationSummaries,
|
||||
cached: true,
|
||||
updatedAt: null,
|
||||
};
|
||||
}
|
||||
const summary = normalizeSummaryText(extracted.text);
|
||||
if (!summary) {
|
||||
throw new Error(
|
||||
"Medication-compliance-status summary was empty after normalization",
|
||||
|
||||
@@ -145,30 +145,120 @@ export function stripJsonFences(content: string): string {
|
||||
}
|
||||
|
||||
/**
|
||||
* Pull the `summary` string out of a model completion. The status
|
||||
* prompts return a `{ "summary": "…" }` envelope; a model that ignores
|
||||
* the contract and returns bare prose falls back to the raw content.
|
||||
* Outcome of reading a status completion for its user-facing summary.
|
||||
*
|
||||
* - `summary` — a `{ "summary": "…" }` envelope parsed to a non-empty
|
||||
* string. Render it.
|
||||
* - `prose` — the model ignored the JSON contract and returned a plain
|
||||
* sentence. Render it verbatim (back-compat with non-JSON-mode providers).
|
||||
* - `unparseable` — the model ATTEMPTED the structured envelope but it came
|
||||
* back malformed, truncated, or summary-less. There is NO safe text to
|
||||
* render: the caller must fall back to its deterministic stub. This is the
|
||||
* state that stops a partial `{ "summary": "…` (braces, property name,
|
||||
* quotes, literal `\n`) from ever reaching the card — the class of bug
|
||||
* reported against the resting-pulse Assessment.
|
||||
*/
|
||||
export type SummaryExtraction =
|
||||
| { kind: "summary"; text: string }
|
||||
| { kind: "prose"; text: string }
|
||||
| { kind: "unparseable" };
|
||||
|
||||
/**
|
||||
* Parse a candidate string as a `{ summary }` object. Reports whether the
|
||||
* string was valid JSON *object* at all (`parsed`) separately from whether it
|
||||
* carried a usable non-empty `summary` — so the caller can tell a malformed /
|
||||
* truncated envelope apart from a well-formed object that simply lacks the
|
||||
* field.
|
||||
*/
|
||||
function parseSummaryObject(candidate: string): {
|
||||
parsed: boolean;
|
||||
summary: string | null;
|
||||
} {
|
||||
try {
|
||||
const obj = JSON.parse(candidate) as unknown;
|
||||
if (obj !== null && typeof obj === "object" && !Array.isArray(obj)) {
|
||||
const value = (obj as { summary?: unknown }).summary;
|
||||
if (typeof value === "string" && value.trim().length > 0) {
|
||||
return { parsed: true, summary: value };
|
||||
}
|
||||
// Well-formed object, but no usable summary (e.g. `{}` or `{summary:""}`).
|
||||
return { parsed: true, summary: null };
|
||||
}
|
||||
// Valid JSON, but not a summary object (array / number / bare string).
|
||||
return { parsed: false, summary: null };
|
||||
} catch {
|
||||
return { parsed: false, summary: null };
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Did the completion OPEN like a JSON object? A genuine prose sentence never
|
||||
* starts with `{` (a stray brace mid-sentence does not count), so a `{`-led
|
||||
* payload that failed both parse attempts above is a malformed / truncated
|
||||
* envelope — never surfaced raw. Strips a leading code fence first so a
|
||||
* fenced-but-truncated object is caught too.
|
||||
*/
|
||||
function looksLikeJsonObjectAttempt(content: string): boolean {
|
||||
const afterFence = content.replace(/^```(?:json)?\s*/i, "").trimStart();
|
||||
return afterFence.startsWith("{");
|
||||
}
|
||||
|
||||
/**
|
||||
* Read a model completion for its user-facing summary and classify the result.
|
||||
*
|
||||
* Tries the content as-is first (the common, fence-free case), then a
|
||||
* fence-stripped retry for providers without a native JSON mode.
|
||||
* fence-stripped retry for providers without a native JSON mode. A completion
|
||||
* that attempted the `{ summary }` envelope but came back malformed / truncated
|
||||
* / summary-less resolves to `unparseable` rather than falling through as raw
|
||||
* text — the caller then serves its deterministic fallback. Only a completion
|
||||
* that never attempted the envelope (genuine prose) passes through verbatim.
|
||||
*/
|
||||
export function extractAssessmentSummary(content: string): SummaryExtraction {
|
||||
const trimmed = content.trim();
|
||||
|
||||
// Attempt 1 — the content is itself a JSON object.
|
||||
const direct = parseSummaryObject(trimmed);
|
||||
if (direct.parsed) {
|
||||
return direct.summary
|
||||
? { kind: "summary", text: direct.summary }
|
||||
: { kind: "unparseable" };
|
||||
}
|
||||
|
||||
// Attempt 2 — strip a code fence / surrounding prose and retry.
|
||||
const stripped = stripJsonFences(trimmed);
|
||||
if (stripped !== trimmed) {
|
||||
const viaFence = parseSummaryObject(stripped);
|
||||
if (viaFence.parsed) {
|
||||
return viaFence.summary
|
||||
? { kind: "summary", text: viaFence.summary }
|
||||
: { kind: "unparseable" };
|
||||
}
|
||||
}
|
||||
|
||||
// Neither parse produced a summary object. If the payload OPENED like an
|
||||
// object, the model attempted the envelope and it came back malformed or
|
||||
// truncated — resolve to the controlled fallback, never the raw braces /
|
||||
// property name / quotes / literal `\n`. Genuine prose falls through.
|
||||
if (looksLikeJsonObjectAttempt(trimmed)) {
|
||||
return { kind: "unparseable" };
|
||||
}
|
||||
return { kind: "prose", text: content };
|
||||
}
|
||||
|
||||
/**
|
||||
* Pull the `summary` string out of a model completion. The status prompts
|
||||
* return a `{ "summary": "…" }` envelope; a model that ignores the contract and
|
||||
* returns bare prose falls back to the raw content. A malformed / truncated /
|
||||
* summary-less structured envelope resolves to an EMPTY string (never the raw
|
||||
* JSON) so no caller can surface partial structured output.
|
||||
*
|
||||
* Prefer `extractAssessmentSummary` in new code: it distinguishes an
|
||||
* unparseable envelope from genuinely empty prose, which this thin wrapper
|
||||
* collapses to `""`.
|
||||
*/
|
||||
export function parseSummaryFromContent(content: string): string {
|
||||
try {
|
||||
const parsed = JSON.parse(content) as { summary?: string };
|
||||
if (typeof parsed.summary === "string") return parsed.summary;
|
||||
} catch {
|
||||
// not directly parseable — try a fence-stripped retry below
|
||||
}
|
||||
try {
|
||||
const stripped = stripJsonFences(content);
|
||||
if (stripped !== content) {
|
||||
const parsed = JSON.parse(stripped) as { summary?: string };
|
||||
if (typeof parsed.summary === "string") return parsed.summary;
|
||||
}
|
||||
} catch {
|
||||
// still not JSON — fall through to the raw content
|
||||
}
|
||||
return content;
|
||||
const extracted = extractAssessmentSummary(content);
|
||||
return extracted.kind === "unparseable" ? "" : extracted.text;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -230,7 +320,8 @@ export async function persistStatusInsight(args: {
|
||||
* withholds.
|
||||
*/
|
||||
export type StatusSummaryOutcome =
|
||||
{ ok: true; text: string } | { ok: false; reason: OutboundReason };
|
||||
| { ok: true; text: string }
|
||||
| { ok: false; reason: OutboundReason | "unparseable_output" };
|
||||
|
||||
/**
|
||||
* The single finalize step every per-metric / biomarker / derived-score status
|
||||
@@ -263,7 +354,14 @@ export function finalizeStatusSummary(
|
||||
content: string,
|
||||
locale: SupportedLocale,
|
||||
): StatusSummaryOutcome {
|
||||
const text = normalizeSummaryText(parseSummaryFromContent(content));
|
||||
const extracted = extractAssessmentSummary(content);
|
||||
// A malformed / truncated / summary-less structured envelope has no safe
|
||||
// text to render. WITHHOLD — the caller serves its deterministic stub. Only
|
||||
// the `unparseable` discriminator is logged, never the assessment content.
|
||||
if (extracted.kind === "unparseable") {
|
||||
return { ok: false, reason: "unparseable_output" };
|
||||
}
|
||||
const text = normalizeSummaryText(extracted.text);
|
||||
if (!text) return { ok: true, text: "" };
|
||||
const decision = screenModelOutput(text, locale, INSIGHTS_CONTRACTS);
|
||||
if (decision.block && decision.reason) {
|
||||
|
||||
Reference in new issue
Block a user