mirror of
https://github.com/MBombeck/HealthLog.git
synced 2026-11-10 13:22:14 +00:00
fix(coach): stop the numeric verifier from flagging correctly-grounded figures
Three confirmed false-positive shapes in findUnverifiedCoachNumbers/
stripUnverifiedNumbers were replacing legitimate prose with the literal
"[unverified]" placeholder even though the underlying data was grounded:
- A correctly-computed delta, midpoint average, or percent change between
two of a payload's own headline aggregate figures (e.g. "down 3.5 kg from
your 7-day average") was never itself a raw leaf in any tool result, so it
was flagged as invented. The authoritative set now also carries the
pairwise delta/average/percent-change between a SINGLE payload's own
named aggregate fields (latest, avg7, avg30, avgSys30, allTimeAvg, ...),
scoped per payload so a weight delta cannot ground itself against an
unrelated metric's figure.
- The always-sent DATA INVENTORY manifest (tool-mode base context) carries a
sample count per domain, but that count was never added to the
authoritative set the way the pinned workout-evidence block already is —
so a plain restatement of a domain's count ("you've logged 42 BP
readings") was flagged whenever the model didn't also call that domain's
retrieval tool in the same turn. The inventory entries now splice into
toolResultPayloads exactly like workoutEvidence.
- A calendar-date ordinal ("July 21st") or a numbered-list marker ("1. Cut
back on sodium") tokenised as a bare number and, when ungrounded, got
soft-stripped mid-word into visibly broken text ("July [unverified]st").
Prose is now masked for these two narrow, high-precision shapes before
numeric extraction; the original text used for stripping is untouched.
A genuinely fabricated figure (unrelated to any grounded point, inventory
count, date, or list marker) is still caught and stripped — covered by new
adversarial cases alongside each fix.
Investigated and ruled out: multi-round tool-loop result accumulation (all
present results across every round already reach the verifier); unit
conversion (the app resolves and serves one consistent unit, so there is no
independent conversion step for the model to get wrong). The Daily
Briefing's separate withhold-a-whole-section grounding gate is unrelated to
the reported symptom (a different failure mode, whole-line omission vs.
inline replacement) and was not touched.
This commit is contained in:
1 parent
62eb431d66
commit
fbdeec9204
5 files changed
+341
-2
No files matched your search
@@ -409,6 +409,76 @@ describe("coach chat — tool-mode routing (F1)", () => {
|
||||
expect(content).not.toContain("138");
|
||||
});
|
||||
|
||||
it("does not flag a sample count restated from the DATA INVENTORY manifest even when its own tool did not run this turn (v1.32.1)", async () => {
|
||||
resolveProviderChain.mockResolvedValue([
|
||||
{ providerType: "anthropic", instance: {} },
|
||||
]);
|
||||
// The DATA INVENTORY (always sent to the model this turn, regardless of
|
||||
// which tools actually fire — see `renderDataInventory`) advertises a
|
||||
// blood-pressure sample count of 42. The model restates it in prose while
|
||||
// the ONLY tool it actually called this turn was for workouts.
|
||||
(buildCoachDataInventory as ReturnType<typeof vi.fn>).mockResolvedValueOnce(
|
||||
{
|
||||
entries: [
|
||||
{
|
||||
tool: "get_metric_series",
|
||||
metric: "bp",
|
||||
domain: "blood pressure",
|
||||
present: true,
|
||||
count: 42,
|
||||
},
|
||||
{
|
||||
tool: "get_workouts",
|
||||
domain: "workouts & training",
|
||||
present: true,
|
||||
count: 12,
|
||||
},
|
||||
],
|
||||
restMode: false,
|
||||
cycleEnabled: false,
|
||||
window: "last30days",
|
||||
probeScope: { sources: ["bp", "workouts"], window: "last30days" },
|
||||
},
|
||||
);
|
||||
const reply =
|
||||
"You've logged 42 blood pressure readings this window, and your last run burned 320 kcal.";
|
||||
(parseKeyValuesSentinel as ReturnType<typeof vi.fn>).mockReturnValue({
|
||||
prose: reply,
|
||||
keyValues: [],
|
||||
malformed: false,
|
||||
malformedEntries: [],
|
||||
});
|
||||
(parseSuggestReminder as ReturnType<typeof vi.fn>).mockReturnValue({
|
||||
prose: reply,
|
||||
});
|
||||
(runCoachToolLoop as ReturnType<typeof vi.fn>).mockImplementation(
|
||||
async () => ({
|
||||
result: { content: reply, tokensUsed: 80, model: "m" },
|
||||
workingProviderType: "anthropic",
|
||||
totalTokens: 80,
|
||||
rounds: 2,
|
||||
toolTrace: [{ name: "get_workouts", present: true }],
|
||||
// ONLY the workouts tool ran — the BP count "42" was never fetched
|
||||
// this turn, it came from the always-sent DATA INVENTORY manifest.
|
||||
toolResults: [
|
||||
{
|
||||
present: true,
|
||||
data: { workouts: [{ type: "run", calories: 320 }] },
|
||||
},
|
||||
],
|
||||
}),
|
||||
);
|
||||
await postAndDrain({ message: "How is my BP and my training?" });
|
||||
const calls = (appendMessage as ReturnType<typeof vi.fn>).mock.calls;
|
||||
const assistantCall = calls.find(
|
||||
(c) => (c[0] as { role: string }).role === "assistant",
|
||||
);
|
||||
const content = (assistantCall?.[0] as { content: string }).content;
|
||||
expect(content).not.toContain("[unverified]");
|
||||
expect(content).toContain("42");
|
||||
expect(content).toContain("320");
|
||||
});
|
||||
|
||||
it("falls back to the snapshot-stuffing path when a provider lacks tools", async () => {
|
||||
resolveProviderChain.mockResolvedValue([
|
||||
{ providerType: "anthropic", instance: {} },
|
||||
|
||||
@@ -653,6 +653,16 @@ async function handleChatRequest(request: NextRequest): Promise<Response> {
|
||||
toolTrace = loop.toolTrace;
|
||||
toolResultPayloads = [
|
||||
...(workoutEvidence === null ? [] : [workoutEvidence]),
|
||||
// v1.32.1 (false-positive closure) — the DATA INVENTORY manifest
|
||||
// (sample counts per domain) rides EVERY tool-mode turn's prompt
|
||||
// regardless of which tools actually ran this turn
|
||||
// (`renderDataInventory`) — it is server-computed and handed to the
|
||||
// model exactly like `workoutEvidence` above, so a plain restatement
|
||||
// of a domain's sample count ("you've logged 42 BP readings") is
|
||||
// grounded even when the model did not also call a retrieval tool
|
||||
// for that domain this turn. Splicing it in here closes that gap
|
||||
// the same way `workoutEvidence` already does.
|
||||
inventory.entries,
|
||||
...(loop.toolResults ?? []).map((r) => r.data),
|
||||
];
|
||||
totalTokensSpent = loop.totalTokens;
|
||||
|
||||
Reference in new issue
Block a user