mirror of
https://github.com/MBombeck/HealthLog.git
synced 2026-11-15 06:52:04 +00:00
Pin where the Coach stops checking its own numbers
Three characterisation tests for grounding gaps found while verifying that
every figure the Coach states comes from the record.
The numeric grounding guard rewrites any figure the turn's Grounding Ledger
cannot account for, and it holds: removing the strip reddens the existing
cross-turn ledger test. What is not covered is its activation. On the tool
path the guard runs only when `loop.toolResults` is non-empty, and a tool that
found nothing resolves to `{ present: false, reason: "none" }` with no
`available` payload, which the loop drops. So the guard disarms itself exactly
when the record is empty: ask about a metric with no readings, and an invented
figure is persisted and streamed verbatim, with no withheld-figure marker to
tell the reader it was never checked. The no-tools snapshot fallback is
populated only in the non-tool-mode branch, so the tool path has nothing to
fall back to, and every cloud provider runs the tool path.
Separately, a rollup bucket outlives the rows it was folded from. The read-swap
falls back to live SQL only on an empty band, `ensureUserRollupsFresh` repairs
the DAY tier over the trailing 90 days only, and it keys off
`Measurement.updatedAt`, which a hard delete does not bump. The integration
test reproduces the hard `deleteMany` in the WHOOP body sync against a
MONTH-band bucket: the deleted value keeps reaching the Coach's coarse tail
while the all-time extremes beside it report only the surviving value, and the
block is stamped current because `asOf` is derived from raw reading age and
knows nothing about rollup age.
All three assert current behaviour and carry a docblock saying so. Closing
either gap will redden them, which is the point.
This commit is contained in:
1 parent
4d53241574
commit
d49479bc64
3 files changed
+614
No files matched your search
@@ -0,0 +1,125 @@
|
||||
/**
|
||||
* CHARACTERISATION — which tool results reach `toolResults`, and why it matters.
|
||||
*
|
||||
* `toolResults` is not just a trace. The chat route derives
|
||||
* `activatingPayloads` from it, and the numeric grounding guard runs ONLY when
|
||||
* that array is non-empty (`insights/chat/route.ts`, the
|
||||
* `activatingPayloads.length > 0` gate). So whatever the loop drops here, the
|
||||
* guard never sees.
|
||||
*
|
||||
* Two shapes of empty read, from `availability.ts` `resolveEmptyRead`:
|
||||
* - nothing recorded at all -> `{ present: false, reason: "none" }`, no `available`
|
||||
* - rows exist out of window -> `{ present: false, reason: "unavailable_in_scope", available }`
|
||||
*
|
||||
* Only the second survives `loop.ts`'s push condition. The first is dropped,
|
||||
* which disarms the grounding guard for that turn. This test pins the drop so
|
||||
* the consequence is visible at the seam where it happens.
|
||||
*/
|
||||
import { describe, expect, it, vi, beforeEach } from "vitest";
|
||||
|
||||
import type { CoachToolResult } from "@/lib/ai/coach/tools/executor";
|
||||
|
||||
const executeCoachTool = vi.fn<(args?: unknown) => Promise<CoachToolResult>>();
|
||||
vi.mock("@/lib/ai/coach/tools/executor", () => ({
|
||||
executeCoachTool: (args: unknown) => executeCoachTool(args),
|
||||
}));
|
||||
|
||||
const runRawCompletionWithFallback = vi.fn();
|
||||
vi.mock("@/lib/ai/provider-runner", () => ({
|
||||
runRawCompletionWithFallback: (args: unknown) =>
|
||||
runRawCompletionWithFallback(args),
|
||||
}));
|
||||
|
||||
import { runCoachToolLoop } from "@/lib/ai/coach/tools/loop";
|
||||
import { COACH_TOOL_DEFS } from "@/lib/ai/coach/tools/definitions";
|
||||
|
||||
const baseArgs = {
|
||||
userId: "u1",
|
||||
providers: [],
|
||||
system: "sys",
|
||||
messages: [{ role: "user" as const, content: "how did I sleep?" }],
|
||||
tools: COACH_TOOL_DEFS,
|
||||
};
|
||||
|
||||
describe("coach tool loop — which results reach the grounding guard", () => {
|
||||
beforeEach(() => {
|
||||
executeCoachTool.mockReset();
|
||||
runRawCompletionWithFallback.mockReset();
|
||||
});
|
||||
|
||||
it("a pure miss (present:false, no available) is dropped from toolResults", async () => {
|
||||
executeCoachTool.mockResolvedValue({ present: false, reason: "none" });
|
||||
runRawCompletionWithFallback
|
||||
.mockResolvedValueOnce({
|
||||
result: {
|
||||
content: "",
|
||||
tokensUsed: 30,
|
||||
model: "mock",
|
||||
providerType: "anthropic" as const,
|
||||
toolCalls: [
|
||||
{ id: "t1", name: "get_sleep", arguments: JSON.stringify({}) },
|
||||
],
|
||||
finishReason: "tool_calls",
|
||||
},
|
||||
workingProvider: { providerType: "anthropic" },
|
||||
fallbackHops: [],
|
||||
})
|
||||
.mockResolvedValueOnce({
|
||||
result: {
|
||||
content: "You averaged 7 h 12 min of sleep.",
|
||||
tokensUsed: 20,
|
||||
model: "mock",
|
||||
providerType: "anthropic" as const,
|
||||
finishReason: "stop",
|
||||
},
|
||||
workingProvider: { providerType: "anthropic" },
|
||||
fallbackHops: [],
|
||||
});
|
||||
|
||||
const loop = await runCoachToolLoop(baseArgs);
|
||||
|
||||
// The trace records that the tool ran and found nothing…
|
||||
expect(loop.toolTrace).toEqual([{ name: "get_sleep", present: false }]);
|
||||
// …but the payload set the route grades against is empty, so the route's
|
||||
// `activatingPayloads.length > 0` gate is false and the numeric grounding
|
||||
// guard never runs for this turn.
|
||||
expect(loop.toolResults).toEqual([]);
|
||||
});
|
||||
|
||||
it("an out-of-window miss WITH `available` is kept, so the guard stays armed", async () => {
|
||||
executeCoachTool.mockResolvedValue({
|
||||
present: false,
|
||||
reason: "unavailable_in_scope",
|
||||
available: { count: 12, reachableWithWindow: "lastYear" },
|
||||
} as unknown as CoachToolResult);
|
||||
runRawCompletionWithFallback
|
||||
.mockResolvedValueOnce({
|
||||
result: {
|
||||
content: "",
|
||||
tokensUsed: 30,
|
||||
model: "mock",
|
||||
providerType: "anthropic" as const,
|
||||
toolCalls: [
|
||||
{ id: "t1", name: "get_sleep", arguments: JSON.stringify({}) },
|
||||
],
|
||||
finishReason: "tool_calls",
|
||||
},
|
||||
workingProvider: { providerType: "anthropic" },
|
||||
fallbackHops: [],
|
||||
})
|
||||
.mockResolvedValueOnce({
|
||||
result: {
|
||||
content: "Nothing in the last 30 days.",
|
||||
tokensUsed: 20,
|
||||
model: "mock",
|
||||
providerType: "anthropic" as const,
|
||||
finishReason: "stop",
|
||||
},
|
||||
workingProvider: { providerType: "anthropic" },
|
||||
fallbackHops: [],
|
||||
});
|
||||
|
||||
const loop = await runCoachToolLoop(baseArgs);
|
||||
expect(loop.toolResults).toHaveLength(1);
|
||||
});
|
||||
});
|
||||
Reference in new issue
Block a user