diff --git a/CHANGELOG.md b/CHANGELOG.md index abd7cd83e..5ec4e1475 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,18 @@ ## [Unreleased] +## [1.32.4] — 2026-07-23 + +- **The Coach's numeric verifier stops flagging figures it was actually given.** + A restated difference, average, or percent change computed from two of the + numbers the model was shown, a sample count restated from the data inventory + that rides every turn, a calendar-date ordinal ("July 21st"), and a + numbered-list marker no longer trip the inline `[unverified]` rewrite that + was appearing in place of correct values. A genuinely fabricated figure — one + with no grounding at all — is still caught and stripped. + +No migrations. No breaking changes. + ## [1.32.3] — 2026-07-23 Security release. It ships on its own so the change to how a second factor is diff --git a/docs/api/openapi.yaml b/docs/api/openapi.yaml index c5b78d02f..0992bcce2 100644 --- a/docs/api/openapi.yaml +++ b/docs/api/openapi.yaml @@ -1,7 +1,7 @@ openapi: 3.1.0 info: title: HealthLog API - version: 1.32.3 + version: 1.32.4 description: >- Self-hosted personal-health-tracking PWA — public API surface for the iOS native client and external ingest. diff --git a/package.json b/package.json index a05dfc191..f989be492 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "healthlog", - "version": "1.32.3", + "version": "1.32.4", "description": "Self-hosted personal-health-tracking PWA with Withings integration, AI insights, and doctor-report PDF export.", "license": "PolyForm-Noncommercial-1.0.0", "homepage": "https://healthlog.dev", diff --git a/public/sw.js b/public/sw.js index f06b7724d..f6fe71355 100644 --- a/public/sw.js +++ b/public/sw.js @@ -36,7 +36,7 @@ try { // v1.4.38.4 → v1.4.42. Do not hand-edit; bump `package.json` and rebuild. const CACHE_VERSION = (typeof self !== "undefined" && self.__APP_VERSION__) || - /* @sw-version-fallback */ "v1.32.3"; + /* @sw-version-fallback */ "v1.32.4"; const STATIC_CACHE = `healthlog-static-${CACHE_VERSION}`; const PAGE_CACHE = `healthlog-pages-${CACHE_VERSION}`; // v1.18.6 — read-only data cache for a curated allowlist of safe GET `/api/*` diff --git a/src/app/api/insights/chat/__tests__/route-tool-mode.test.ts b/src/app/api/insights/chat/__tests__/route-tool-mode.test.ts index b2cd3d496..3e99f2cf6 100644 --- a/src/app/api/insights/chat/__tests__/route-tool-mode.test.ts +++ b/src/app/api/insights/chat/__tests__/route-tool-mode.test.ts @@ -409,6 +409,135 @@ describe("coach chat — tool-mode routing (F1)", () => { expect(content).not.toContain("138"); }); + it("does not flag a sample count restated from the DATA INVENTORY manifest even when its own tool did not run this turn (v1.32.1)", async () => { + resolveProviderChain.mockResolvedValue([ + { providerType: "anthropic", instance: {} }, + ]); + // The DATA INVENTORY (always sent to the model this turn, regardless of + // which tools actually fire — see `renderDataInventory`) advertises a + // blood-pressure sample count of 42. The model restates it in prose while + // the ONLY tool it actually called this turn was for workouts. + (buildCoachDataInventory as ReturnType).mockResolvedValueOnce( + { + entries: [ + { + tool: "get_metric_series", + metric: "bp", + domain: "blood pressure", + present: true, + count: 42, + }, + { + tool: "get_workouts", + domain: "workouts & training", + present: true, + count: 12, + }, + ], + restMode: false, + cycleEnabled: false, + window: "last30days", + probeScope: { sources: ["bp", "workouts"], window: "last30days" }, + }, + ); + const reply = + "You've logged 42 blood pressure readings this window, and your last run burned 320 kcal."; + (parseKeyValuesSentinel as ReturnType).mockReturnValue({ + prose: reply, + keyValues: [], + malformed: false, + malformedEntries: [], + }); + (parseSuggestReminder as ReturnType).mockReturnValue({ + prose: reply, + }); + (runCoachToolLoop as ReturnType).mockImplementation( + async () => ({ + result: { content: reply, tokensUsed: 80, model: "m" }, + workingProviderType: "anthropic", + totalTokens: 80, + rounds: 2, + toolTrace: [{ name: "get_workouts", present: true }], + // ONLY the workouts tool ran — the BP count "42" was never fetched + // this turn, it came from the always-sent DATA INVENTORY manifest. + toolResults: [ + { + present: true, + data: { workouts: [{ type: "run", calories: 320 }] }, + }, + ], + }), + ); + await postAndDrain({ message: "How is my BP and my training?" }); + const calls = (appendMessage as ReturnType).mock.calls; + const assistantCall = calls.find( + (c) => (c[0] as { role: string }).role === "assistant", + ); + const content = (assistantCall?.[0] as { content: string }).content; + expect(content).not.toContain("[unverified]"); + expect(content).toContain("42"); + expect(content).toContain("320"); + }); + + it("keeps the verifier dormant when NO tool returned figures this turn, even though the inventory rides the prompt (v1.32.1 regression)", async () => { + // Regression guard for the integration failure: in tool mode the model can + // answer a quick turn WITHOUT calling any tool (empty toolResults). The + // always-sent DATA INVENTORY carries counts only, never the snapshot's + // pre-computed averages. The verifier must NOT activate off the inventory + // alone — otherwise a figure the model narrated from the snapshot (a + // 30-day systolic average) is wrongly stripped. Matches `main`'s "no tool + // figures → skip, prompt-level grounding is the backstop" contract. + resolveProviderChain.mockResolvedValue([ + { providerType: "anthropic", instance: {} }, + ]); + (buildCoachDataInventory as ReturnType).mockResolvedValueOnce( + { + entries: [ + { + tool: "get_metric_series", + metric: "bp", + domain: "blood pressure", + present: true, + count: 14, + }, + ], + restMode: false, + cycleEnabled: false, + window: "last30days", + probeScope: { sources: ["bp"], window: "last30days" }, + }, + ); + const reply = "Your 30-day systolic average is 130 mmHg, trending up."; + (parseKeyValuesSentinel as ReturnType).mockReturnValue({ + prose: reply, + keyValues: [], + malformed: false, + malformedEntries: [], + }); + (parseSuggestReminder as ReturnType).mockReturnValue({ + prose: reply, + }); + (runCoachToolLoop as ReturnType).mockImplementation( + async () => ({ + result: { content: reply, tokensUsed: 60, model: "m" }, + workingProviderType: "anthropic", + totalTokens: 60, + rounds: 1, + toolTrace: [], + // No tool returned figures this turn — the model answered directly. + toolResults: [], + }), + ); + await postAndDrain({ message: "How is my blood pressure this month?" }); + const calls = (appendMessage as ReturnType).mock.calls; + const assistantCall = calls.find( + (c) => (c[0] as { role: string }).role === "assistant", + ); + const content = (assistantCall?.[0] as { content: string }).content; + expect(content).not.toContain("[unverified]"); + expect(content).toContain("130"); + }); + it("falls back to the snapshot-stuffing path when a provider lacks tools", async () => { resolveProviderChain.mockResolvedValue([ { providerType: "anthropic", instance: {} }, diff --git a/src/app/api/insights/chat/route.ts b/src/app/api/insights/chat/route.ts index ad5fef4fb..718a0a130 100644 --- a/src/app/api/insights/chat/route.ts +++ b/src/app/api/insights/chat/route.ts @@ -651,10 +651,28 @@ async function handleChatRequest(request: NextRequest): Promise { result = loop.result; workingProviderType = loop.workingProviderType; toolTrace = loop.toolTrace; - toolResultPayloads = [ + // v1.32.1 — the numeric verifier ACTIVATES only when this turn actually + // delivered figures the model was told to ground against: a pinned + // workout-evidence block or a present tool result. The DATA INVENTORY + // manifest (sample counts per domain) is NOT an activator — it rides + // every tool-mode prompt even on a turn where the model answered + // without calling a tool, and on that turn the base prompt deliberately + // carries no pre-computed figures (the model must fetch them), so the + // verifier must stay dormant and leave the prompt-level grounding rule + // as the backstop, exactly as on `main`. Activating it off the + // counts-only inventory would flag a snapshot figure the model cited + // without a fresh tool call as ungrounded (a real regression caught by + // the integration suite). When the turn IS active, the inventory counts + // still WIDEN the authoritative set so a plain count restatement + // ("you've logged 42 BP readings") stays grounded. + const presentToolPayloads = [ ...(workoutEvidence === null ? [] : [workoutEvidence]), ...(loop.toolResults ?? []).map((r) => r.data), ]; + toolResultPayloads = + presentToolPayloads.length > 0 + ? [...presentToolPayloads, inventory.entries] + : []; totalTokensSpent = loop.totalTokens; cachedTokensSpent = loop.cachedTokens; } else { diff --git a/src/lib/ai/coach/__tests__/coach-prose-grounding.test.ts b/src/lib/ai/coach/__tests__/coach-prose-grounding.test.ts index c6a61a988..d9150d489 100644 --- a/src/lib/ai/coach/__tests__/coach-prose-grounding.test.ts +++ b/src/lib/ai/coach/__tests__/coach-prose-grounding.test.ts @@ -3,6 +3,14 @@ * a number the model cited that no tool returned this turn (transcription * drift), exempt structural window/ordinal integers, and no-op when no tool * returned figures. Soft-strip replaces only the flagged tokens. + * + * v1.32.1 — closes three real-world false-positive shapes reported against + * grounded Coach replies: a correctly-computed derived delta/average/percent + * between two of a payload's own headline figures, a calendar-date ordinal + * ("July 21st") mangled mid-word into "[unverified]st", and a numbered-list + * recommendation line ("1. Cut back on sodium") whose marker got flagged as a + * bare figure. Each new case is paired with an adversarial case proving a + * genuine fabrication in the SAME shape is still caught. */ import { describe, expect, it } from "vitest"; @@ -50,6 +58,44 @@ describe("findUnverifiedCoachNumbers", () => { expect(findUnverifiedCoachNumbers(prose, payloads)).toEqual([]); }); + it("grounds a 30-day systolic average against the real tool section that carries it", () => { + // Mirrors the integration fixture: the model narrates the pre-computed + // 30-day systolic average (130) that the bp tool section actually returned. + // It must NOT be flagged. (Regression companion: the route must feed this + // real section, not a counts-only inventory — see route-tool-mode tests.) + const bpSection = [ + { metric: "bp", section: { aggregate: { avgSys30: 130 } } }, + ]; + const prose = "Your 30-day systolic average is 130 mmHg, trending up."; + expect(findUnverifiedCoachNumbers(prose, bpSection)).toEqual([]); + }); + + it("does NOT ground an average against a counts-only inventory payload (documents why the route must not activate on the inventory alone)", () => { + // The DATA INVENTORY carries per-domain sample COUNTS, never the snapshot's + // averages. If the route were to feed inventory-entries alone as the + // authoritative set, a cited average (130) would be graded against {14} + // and wrongly flagged. This asserts that shape so the route's activation + // gate (only run when a real tool result / workout block is present) is + // load-bearing, not incidental. + const inventoryOnly = [ + [ + { + tool: "get_metric_series", + metric: "bp", + domain: "blood pressure", + present: true, + count: 14, + }, + ], + ]; + const findings = findUnverifiedCoachNumbers( + "Your 30-day systolic average is 130 mmHg.", + inventoryOnly, + ); + expect(findings).toHaveLength(1); + expect(findings[0].value).toBe(130); + }); + it("exempts structural window + small-ordinal integers", () => { const prose = "Over the last 30 days, 2 readings stood out of 7 total."; // 30 + 7 are window integers; 2 is a small ordinal — none graded. @@ -68,6 +114,137 @@ describe("findUnverifiedCoachNumbers", () => { it("returns empty on empty prose", () => { expect(findUnverifiedCoachNumbers("", payloads)).toEqual([]); }); + + describe("derived arithmetic (delta / average / percent)", () => { + // A realistic weight tool-result payload: the model was given latest, + // avg7 and avg30 — never a pre-computed delta (unlike the Daily + // Briefing's `signalsOfDay`), so a correctly-computed "down 3.5 kg from + // your 7-day average" is a legitimate derivation the checker must not + // flag. + const weightPayload = [ + { + metric: "weight", + section: { aggregate: { latest: 72.4, avg7: 75.9, avg30: 74.1 } }, + }, + ]; + + it("does not flag a correctly-computed delta between two grounded aggregate points", () => { + const prose = + "You're down 3.5 kg from your 7-day average of 75.9 kg, now at 72.4 kg — nice trend."; + expect(findUnverifiedCoachNumbers(prose, weightPayload)).toEqual([]); + }); + + it("does not flag a correctly-computed midpoint average of two grounded aggregate points", () => { + const prose = + "Your latest (72.4 kg) and 30-day average (74.1 kg) put you around 73.25 kg overall."; + expect(findUnverifiedCoachNumbers(prose, weightPayload)).toEqual([]); + }); + + it("does not flag a correctly-computed percent change between two grounded aggregate points", () => { + // (75.9 - 72.4) / 75.9 * 100 ≈ 4.61% + const prose = "That's roughly a 4.6% drop from your 7-day average."; + expect(findUnverifiedCoachNumbers(prose, weightPayload)).toEqual([]); + }); + + it("still flags a fabricated figure that is NOT a derived value of the grounded points", () => { + const prose = + "Your latest reading was 72.4 kg, and I estimate you'll hit 55.0 kg by next month."; + const findings = findUnverifiedCoachNumbers(prose, weightPayload); + expect(findings).toHaveLength(1); + expect(findings[0].value).toBe(55); + }); + + it("does not let a weight delta ground itself against an unrelated metric's figure (per-payload scoping)", () => { + // Blood pressure's own aggregate lives in a SEPARATE tool-result + // payload; a "difference" here must derive only from ITS OWN points, + // never pool with the unrelated weight payload above. + const bpAndWeight = [ + ...weightPayload, + { + metric: "bp", + section: { aggregate: { avgSys30: 128, avgDia30: 82 } }, + }, + ]; + // 46 is |128 - 82| — a legitimate BP pulse-pressure-shaped delta — + // grounded via the bp payload's own points, not a coincidence. + expect( + findUnverifiedCoachNumbers( + "Your pulse pressure is about 46.", + bpAndWeight, + ), + ).toEqual([]); + }); + }); + + describe("date ordinals and numbered-list markers", () => { + const bpPayload = [ + { + metric: "bp", + section: { aggregate: { latest: 138, avg7: 132, avg30: 128 } }, + }, + ]; + + it("does not flag a day-of-month ordinal even though the bare day number is ungrounded", () => { + const prose = + "Your blood pressure spiked to 138 on July 21st, well above your typical range."; + expect(findUnverifiedCoachNumbers(prose, bpPayload)).toEqual([]); + }); + + it("does not corrupt the ordinal suffix when soft-stripping the rest of the reply", () => { + // Regression guard for the exact mangling reported: "[unverified]st". + const prose = + "Your blood pressure spiked to 138 on July 21st, well above your typical range."; + const findings = findUnverifiedCoachNumbers(prose, bpPayload); + const { prose: out } = stripUnverifiedNumbers(prose, findings); + expect(out).not.toContain("[unverified]st"); + expect(out).toBe(prose); + }); + + it("does not flag numbered-list recommendation markers", () => { + // Item numbers run past 3 (the pre-existing small-ordinal exemption) so + // this genuinely exercises the line-start list-marker mask, not the + // unrelated small-integer rule. + const prose = + "Your systolic hit 138 today. A few ideas:\n1. Cut back on sodium\n2. Take a short walk after meals\n3. Log your next reading tomorrow\n4. Recheck in the evening\n5. Note any stress today"; + expect(findUnverifiedCoachNumbers(prose, bpPayload)).toEqual([]); + }); + + it("still flags a fabricated figure sitting right next to a date ordinal", () => { + const prose = "On July 21st your systolic hit 199, a new high."; + const findings = findUnverifiedCoachNumbers(prose, bpPayload); + expect(findings).toHaveLength(1); + expect(findings[0].value).toBe(199); + }); + + it("still flags a genuine leading figure that happens to open a line (not list-shaped without the trailing space)", () => { + // "199kg" has no space after the number at all — never mistaken for a + // "N. " list marker, which requires a literal dot-then-space. + const prose = "199kg would be an alarming reading."; + const findings = findUnverifiedCoachNumbers(prose, bpPayload); + expect(findings).toHaveLength(1); + expect(findings[0].value).toBe(199); + }); + + it("still flags a decimal figure that opens a line (a digit, not whitespace, follows the dot)", () => { + // "199.5" at line-start is a genuine decimal reading, not a "N. " list + // marker — the masking regex only fires when whitespace follows the + // dot, so this stays fully checked. + const prose = "199.5 would be an alarming systolic reading."; + const findings = findUnverifiedCoachNumbers(prose, bpPayload); + expect(findings).toHaveLength(1); + expect(findings[0].value).toBe(199.5); + }); + }); + + it("still flags a drifted number when the prose also derives / dates cleanly (adversarial floor)", () => { + // The classic P6 regression case must survive every new exemption. + const findings = findUnverifiedCoachNumbers( + "Your systolic averaged about 138 recently.", + payloads, + ); + expect(findings).toHaveLength(1); + expect(findings[0].value).toBe(138); + }); }); describe("stripUnverifiedNumbers", () => { diff --git a/src/lib/ai/coach/coach-prose-grounding.ts b/src/lib/ai/coach/coach-prose-grounding.ts index 49efb46f5..e3052cd39 100644 --- a/src/lib/ai/coach/coach-prose-grounding.ts +++ b/src/lib/ai/coach/coach-prose-grounding.ts @@ -95,6 +95,114 @@ function isGrounded( return false; } +/** + * v1.32.1 (false-positive closure) — the small, named "central tendency" + * fields every snapshot block already exposes (`latest`, `avg7`, `avg30`, + * `avgSys30`, `allTimeAvg`, …). A model correctly narrating "down 3.5 kg from + * your 7-day average" is computing a DELTA between two of these — a + * legitimate derivation, not a fabrication — but the raw figure `3.5` is not + * itself a leaf anywhere in the payload, so the plain leaf-matching above + * flags it. Deliberately an ALLOWLIST of field names, not "every numeric + * leaf": a whole month of daily timeline samples would blow the pairwise + * derivation up combinatorially (see `deriveArithmeticValues`) and start + * matching a genuinely fabricated figure by coincidence, which is exactly the + * hole this checker exists to close. A raw sample, a count, or a day-offset + * never matches this pattern. + */ +const AGGREGATE_POINT_KEY = + /^(?:latest|mean|median|avg(?:Sys|Dia)?\d*(?:LastMonth|LastYear)?|allTime(?:Avg|Min|Max)(?:Sys|Dia)?)$/; + +/** + * Defensive cap on how many aggregate points one payload may contribute to + * the pairwise derivation below. The named fields the allowlist targets are a + * handful per metric block (2-6); a payload that somehow surfaces more than + * this is not the small aggregate shape the heuristic targets, so skip + * deriving from it rather than pay an unbounded O(n^2). + */ +const MAX_AGGREGATE_POINTS = 12; + +/** Collect every numeric leaf reachable under an `AGGREGATE_POINT_KEY` field. */ +function collectAggregatePoints( + value: unknown, + out: Set, + keyHint?: string, +): void { + if (value === null || value === undefined) return; + if (typeof value === "number") { + if ( + Number.isFinite(value) && + keyHint !== undefined && + AGGREGATE_POINT_KEY.test(keyHint) + ) { + out.add(value); + } + return; + } + if (Array.isArray(value)) { + for (const item of value) collectAggregatePoints(item, out, keyHint); + return; + } + if (typeof value === "object") { + for (const [k, v] of Object.entries(value as Record)) { + collectAggregatePoints(v, out, k); + } + } +} + +/** + * The delta, midpoint average, and percent-change between every PAIR of a + * SINGLE payload's own aggregate points — the shapes a model legitimately + * narrates ("down 3.5 kg from your 7-day average", "roughly a 5% drop"). + * Scoped per payload (one tool result / the pinned workout block / one + * inventory row), never pooled across the whole turn's authoritative set, so + * a weight delta cannot ground itself against an unrelated blood-pressure + * figure that happens to land nearby. + */ +function deriveArithmeticValues(payload: unknown): number[] { + const points = new Set(); + collectAggregatePoints(payload, points); + const arr = [...points]; + if (arr.length < 2 || arr.length > MAX_AGGREGATE_POINTS) return []; + const derived: number[] = []; + for (let i = 0; i < arr.length; i += 1) { + for (let j = i + 1; j < arr.length; j += 1) { + const a = arr[i]; + const b = arr[j]; + const diff = Math.abs(a - b); + derived.push(diff); + derived.push((a + b) / 2); + if (a !== 0) derived.push((diff / Math.abs(a)) * 100); + if (b !== 0) derived.push((diff / Math.abs(b)) * 100); + } + } + return derived; +} + +/** + * v1.32.1 (false-positive closure) — mask date-ordinal and numbered-list + * markers OUT of a working copy of the prose before it is tokenised for + * numbers, so "on July 21st" or a "1. Reduce sodium" recommendation line never + * reads as a restated health figure. `extractNumbers` has no notion of + * surrounding context, so this pre-processing happens on a throwaway copy — + * the ORIGINAL prose (and hence `stripUnverifiedNumbers`, which edits the real + * text via the flagged token) is untouched. Deliberately high precision: + * - a number immediately followed by an English ordinal suffix (21st, 3rd, + * 142nd) is a day-of-month or a rank, never a restated magnitude; + * - a number at the START of a line followed by ". " (a numbered-list + * marker) is masked, but ONLY at line-start with a space right after the + * dot — "Systolic 128. Blood pressure remained stable." keeps its 128 + * because that number does not open its own line, and "3.5 kg" keeps its + * figure because a digit (not whitespace) follows the dot. + */ +function extractClaimableNumbers( + prose: string, +): Array<{ value: number; raw: string }> { + const masked = prose + .replace(/\b\d{1,3}(?:st|nd|rd|th)\b/gi, "DATE") + .replace(/^(\s*)\d{1,3}\.(\s)/gm, "$1LIST$2"); + return extractNumbers(masked); +} + /** * Find every number the Coach prose asserts that does not trace to a figure * returned by a tool this turn. Returns an empty array when there is no prose, @@ -109,14 +217,19 @@ export function findUnverifiedCoachNumbers( ): UnverifiedCoachNumber[] { if (!prose) return []; const authoritative = new Set(); - for (const payload of toolPayloads) + for (const payload of toolPayloads) { collectNumericLeaves(payload, authoritative); + // Widen the authoritative set with the derived figures a model may + // legitimately compute from this payload's own headline numbers, without + // pooling across unrelated payloads (see `deriveArithmeticValues`). + for (const d of deriveArithmeticValues(payload)) authoritative.add(d); + } // No authoritative figures (a qualitative turn / no-tools path) — nothing to // grade against. The prompt-level grounding rule remains the backstop. if (authoritative.size === 0) return []; const findings: UnverifiedCoachNumber[] = []; - for (const { value, raw } of extractNumbers(prose)) { + for (const { value, raw } of extractClaimableNumbers(prose)) { if (isStructural(value, raw)) continue; if (isGrounded(value, authoritative)) continue; findings.push({ value, source: raw.slice(0, 32) });