Pin where the Coach stops checking its own numbers

Three characterisation tests for grounding gaps found while verifying that
every figure the Coach states comes from the record.

The numeric grounding guard rewrites any figure the turn's Grounding Ledger
cannot account for, and it holds: removing the strip reddens the existing
cross-turn ledger test. What is not covered is its activation. On the tool
path the guard runs only when `loop.toolResults` is non-empty, and a tool that
found nothing resolves to `{ present: false, reason: "none" }` with no
`available` payload, which the loop drops. So the guard disarms itself exactly
when the record is empty: ask about a metric with no readings, and an invented
figure is persisted and streamed verbatim, with no withheld-figure marker to
tell the reader it was never checked. The no-tools snapshot fallback is
populated only in the non-tool-mode branch, so the tool path has nothing to
fall back to, and every cloud provider runs the tool path.

Separately, a rollup bucket outlives the rows it was folded from. The read-swap
falls back to live SQL only on an empty band, `ensureUserRollupsFresh` repairs
the DAY tier over the trailing 90 days only, and it keys off
`Measurement.updatedAt`, which a hard delete does not bump. The integration
test reproduces the hard `deleteMany` in the WHOOP body sync against a
MONTH-band bucket: the deleted value keeps reaching the Coach's coarse tail
while the all-time extremes beside it report only the surviving value, and the
block is stamped current because `asOf` is derived from raw reading age and
knows nothing about rollup age.

All three assert current behaviour and carry a docblock saying so. Closing
either gap will redden them, which is the point.
This commit is contained in:
Marc-André Bombeck committed 2026-08-22 16:14:54 +02:00
1 parent 4d53241574
commit d49479bc64
3 files changed
+614

No files matched your search

@@ -0,0 +1,356 @@
import { describe, it, expect, vi, beforeEach } from "vitest";
/**
* CHARACTERISATION — when the Coach's numeric grounding guard is armed, and
* when it is not.
*
* The guard rewrites any figure in the reply that the turn's Grounding Ledger
* cannot account for. It runs only when `activatingPayloads` is non-empty, and
* on the tool path that array comes solely from `loop.toolResults`. A tool that
* found nothing at all resolves to `{ present: false, reason: "none" }` with no
* `available` payload, and `loop.ts` drops that shape from `toolResults`
* (pinned in `tools-loop-miss-payloads.test.ts`). The no-tools snapshot
* fallback is populated only in the non-tool-mode branch, so on the tool path
* there is nothing to fall back to.
*
* The consequence, pinned below: on a tool-mode turn where every tool missed —
* i.e. exactly when the user has no data for what they asked about — a figure
* the model invented is streamed and persisted verbatim, and the reply carries
* no `unverifiedFigures` marker, so it reads to the user as fully checked.
*
* These tests assert the CURRENT behaviour, not the desired one. If the
* activation gate is widened so the guard covers the all-missed turn, they will
* fail — that failure is the fix landing, and the assertions should be flipped
* to the safe expectation at that point.
*
* The first test is the control: with one present tool result the guard is
* armed and the same invented figure is stripped.
*/
const SNAPSHOT_JSON = '{"bp":{"aggregate":{"mean":128}}}';
vi.mock("@/lib/api-handler", () => ({
apiHandler: <T extends (...args: unknown[]) => unknown>(fn: T) => fn,
requireAuth: vi.fn(async () => ({ user: { id: "u1", locale: "en" } })),
HttpError: class HttpError extends Error {
status: number;
constructor(status: number, message: string) {
super(message);
this.status = status;
}
},
}));
vi.mock("@/lib/modules/gate", () => ({
requireModuleEnabled: vi.fn(async () => ({ enabled: true })),
isModuleEnabled: vi.fn(async () => true),
}));
vi.mock("@/lib/feature-flags", () => ({
requireAssistantSurface: vi.fn(async () => undefined),
}));
vi.mock("@/lib/api-response", () => ({
apiError: (error: string, status: number) => ({ data: null, error, status }),
apiSuccess: (data: unknown) => ({ data, error: null, status: 200 }),
}));
vi.mock("@/lib/logging/context", () => ({
annotate: vi.fn(),
getEvent: vi.fn(),
}));
vi.mock("@/lib/logging/redact", () => ({
redactSecrets: (s: string) => s,
redactOptional: (s: unknown) => s,
}));
vi.mock("@/lib/auth/audit", () => ({ auditLog: vi.fn() }));
vi.mock("@/lib/db", () => ({
prisma: {
user: { findUnique: vi.fn(async () => ({ coachPrefsJson: null })) },
coachConversation: { findFirst: vi.fn(async () => ({ id: "c1" })) },
},
}));
const { checkRateLimit } = vi.hoisted(() => ({
checkRateLimit: vi.fn(async () => ({ allowed: true })),
}));
vi.mock("@/lib/rate-limit", () => ({ checkRateLimit }));
vi.mock("@/lib/i18n/server-locale", () => ({
resolveServerLocale: vi.fn(async () => "en"),
}));
const { runStreamingRawCompletionWithFallback } = vi.hoisted(() => ({
runStreamingRawCompletionWithFallback: vi.fn(),
}));
vi.mock("@/lib/ai/provider-runner", () => ({
AllProvidersFailedError: class extends Error {},
runStreamingRawCompletionWithFallback,
}));
const { resolveProviderChain } = vi.hoisted(() => ({
resolveProviderChain: vi.fn(),
}));
vi.mock("@/lib/ai/provider", () => ({
resolveProviderChain,
resolveProvider: vi.fn(),
}));
const { assertConsentForChain } = vi.hoisted(() => ({
assertConsentForChain: vi.fn(async () => undefined),
}));
vi.mock("@/lib/ai/consent-guard", () => ({ assertConsentForChain }));
vi.mock("@/lib/ai/prompts/insight-generator", () => ({ PROMPT_VERSION: "x" }));
vi.mock("@/lib/ai/ai-budgets", () => ({
AI_BUDGETS: { coach: { maxTokens: 1500, temperature: 0.4 } },
}));
vi.mock("@/lib/ai/coach/types", async () => {
const actual = await vi.importActual<typeof import("@/lib/ai/coach/types")>(
"@/lib/ai/coach/types",
);
return actual;
});
const { fetchConversationWithMessages } = vi.hoisted(() => ({
fetchConversationWithMessages: vi.fn(),
}));
vi.mock("@/lib/ai/coach/persistence", () => ({
appendMessage: vi.fn(async () => ({ id: "m1" })),
createConversation: vi.fn(async () => ({ id: "c1" })),
fetchConversationWithMessages,
listConversations: vi.fn(),
}));
vi.mock("@/lib/ai/coach/coach-memory-shared", () => ({
enqueueCoachMemoryRefresh: vi.fn(),
}));
vi.mock("@/lib/ai/coach/facts", () => ({
storeDeterministicFacts: vi.fn(async () => undefined),
}));
// Guard II — the schedule read. Empty here; the schedule-gated dose rule is
// unit-tested in the outbound-screen suite.
vi.mock("@/lib/medications/scheduled-doses", () => ({
getScheduledDoseValues: vi.fn(async () => []),
}));
const { reserveBudget, reconcileSpend } = vi.hoisted(() => ({
reserveBudget: vi.fn(async () => ({ allowed: true, reserved: 3000 })),
reconcileSpend: vi.fn(async () => undefined),
}));
vi.mock("@/lib/ai/coach/budget", () => ({
buildDateKey: vi.fn(() => "2026-07-23"),
reserveBudget,
reconcileSpend,
resolveDailyCap: vi.fn(() => 2_000_000),
}));
const { detectRefusal } = vi.hoisted(() => ({
detectRefusal: vi.fn(() => ({ refuse: false })),
}));
vi.mock("@/lib/ai/coach/refusal", () => ({ detectRefusal }));
vi.mock("@/lib/ai/coach/outbound-guard", () => ({
screenCoachReply: vi.fn(() => ({ block: false })),
coachOutboundFallback: vi.fn(() => "fallback"),
}));
vi.mock("@/lib/ai/coach/system-prompt", () => ({
getCoachSystemPrompt: vi.fn(() => "SYSTEM"),
}));
vi.mock("@/lib/ai/coach/about-me", () => ({
getSelfContextTextForUser: vi.fn(async () => null),
}));
vi.mock("@/lib/ai/coach/snapshot", () => ({
buildCoachSnapshot: vi.fn(async () => ({
snapshotJson: SNAPSHOT_JSON,
sections: { bloodPressure: { aggregate: { mean: 128 } } },
provenance: { windows: ["last30days"], metrics: ["bp"] },
referenceGrounding: "REFERENCE RANGES",
})),
}));
vi.mock("@/lib/workouts/hr-series", () => ({
buildWorkoutHrSeries: vi.fn(async () => null),
}));
vi.mock("@/lib/workouts/zones", () => ({
computeZones: vi.fn(() => null),
hrMaxFromAge: vi.fn(() => 185),
parseWhoopZoneDurations: vi.fn(() => null),
}));
vi.mock("@/lib/workouts/sport-context", () => ({
buildSportContext: vi.fn(async () => null),
}));
const {
buildCoachDataInventory,
renderDataInventory,
renderFocusHint,
runCoachToolLoop,
} = vi.hoisted(() => ({
buildCoachDataInventory: vi.fn(async () => ({
entries: [],
restMode: false,
cycleEnabled: false,
window: "last30days",
probeScope: { sources: ["bp"], window: "last30days" },
})),
renderDataInventory: vi.fn(() => "DATA INVENTORY\n- blood pressure: present"),
renderFocusHint: vi.fn(() => ""),
runCoachToolLoop: vi.fn(),
}));
vi.mock("@/lib/ai/coach/tools", () => ({
COACH_TOOL_DEFS: [{ name: "get_metric_series" }],
MAX_ROUNDS: 3,
buildCoachDataInventory,
renderDataInventory,
renderFocusHint,
buildToolModeAddendum: vi.fn(() => "TOOL ADDENDUM"),
runCoachToolLoop,
}));
const { parseKeyValuesSentinel } = vi.hoisted(() => ({
parseKeyValuesSentinel: vi.fn(),
}));
vi.mock("@/lib/ai/coach/keyvalues", () => ({ parseKeyValuesSentinel }));
const { parseSuggestReminder } = vi.hoisted(() => ({
parseSuggestReminder: vi.fn(),
}));
vi.mock("@/lib/ai/coach/suggest-reminder", () => ({ parseSuggestReminder }));
vi.mock("@/lib/ai/coach/suggest-gate", () => ({ gateSuggestion: vi.fn() }));
vi.mock("@/lib/validations/coach-prefs", () => ({
parseCoachPrefs: vi.fn(() => ({ defaultWindow: undefined })),
DEFAULT_REMINDER_SUGGESTION_PREFS: {},
}));
const { appendMessage } = await import("@/lib/ai/coach/persistence");
const sse = vi.hoisted(() => ({ done: Promise.resolve() as Promise<unknown> }));
vi.mock("@/lib/sse/create-stream", () => ({
createSseStream: (
producer: (c: {
signal: { aborted: boolean };
enqueue: () => void;
}) => void | Promise<void>,
) => {
sse.done = Promise.resolve(
producer({ signal: { aborted: false }, enqueue: () => {} }),
);
return new ReadableStream();
},
}));
import { POST } from "../route";
const post = POST as unknown as (req: Request) => Promise<Response>;
function chatReq(body: Record<string, unknown>): Request {
return new Request("http://localhost/api/insights/chat", {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify(body),
});
}
/** Force the model's assembled prose through the sentinel/suggest parsers. */
function stubReply(prose: string, toolResults: unknown[]): void {
parseKeyValuesSentinel.mockReturnValue({
prose,
keyValues: [],
malformed: false,
malformedEntries: [],
});
parseSuggestReminder.mockReturnValue({ prose });
runCoachToolLoop.mockImplementation(async () => ({
result: { content: prose, tokensUsed: 80, model: "m" },
workingProviderType: "anthropic",
totalTokens: 80,
rounds: 1,
toolTrace: [{ name: "get_metric_series", present: true }],
toolResults,
}));
}
function assistantContent(): string {
const calls = (appendMessage as ReturnType<typeof vi.fn>).mock.calls;
const assistant = calls.find(
(c) => (c[0] as { role: string }).role === "assistant",
);
return (assistant?.[0] as { content: string }).content;
}
function assistantProvenance(): Record<string, unknown> | null | undefined {
const calls = (appendMessage as ReturnType<typeof vi.fn>).mock.calls;
const assistant = calls.find(
(c) => (c[0] as { role: string }).role === "assistant",
);
return (
assistant?.[0] as {
metricSource?: Record<string, unknown> | null;
}
).metricSource;
}
describe("coach chat — grounding-guard activation", () => {
beforeEach(() => {
vi.clearAllMocks();
reserveBudget.mockResolvedValue({ allowed: true, reserved: 3000 });
detectRefusal.mockReturnValue({ refuse: false });
checkRateLimit.mockResolvedValue({ allowed: true });
assertConsentForChain.mockResolvedValue(undefined);
resolveProviderChain.mockResolvedValue([
{ providerType: "anthropic", instance: {} },
]);
fetchConversationWithMessages.mockResolvedValue({
id: "c1",
attachmentCount: 0,
summary: null,
messages: [],
});
});
it("CONTROL — one present tool result arms the guard: the invented figure is stripped", async () => {
stubReply("Your sleep averaged 432 minutes last week.", [
{ present: true, data: { metric: "bp", aggregate: { avgSys30: 128 } } },
]);
await post(chatReq({ conversationId: "c1", message: "how did I sleep?" }));
await sse.done;
expect(assistantContent()).not.toContain("432");
expect(assistantContent()).toContain("[…]");
expect(assistantProvenance()?.unverifiedFigures).toBe(1);
});
it("every tool missed: the guard stays dormant and the invented figure survives", async () => {
// A pure miss never reaches `toolResults` (see tools-loop-miss-payloads),
// so this is what the route sees when the record holds nothing.
stubReply("Your sleep averaged 432 minutes last week.", []);
await post(chatReq({ conversationId: "c1", message: "how did I sleep?" }));
await sse.done;
expect(assistantContent()).toBe(
"Your sleep averaged 432 minutes last week.",
);
// No withheld-figure notice either, so the reply reads as verified.
expect(assistantProvenance()?.unverifiedFigures).toBeUndefined();
});
it("the model called no tool at all: the guard stays dormant", async () => {
const prose = "Your resting heart rate has been averaging 58 bpm.";
parseKeyValuesSentinel.mockReturnValue({
prose,
keyValues: [],
malformed: false,
malformedEntries: [],
});
parseSuggestReminder.mockReturnValue({ prose });
runCoachToolLoop.mockImplementation(async () => ({
result: { content: prose, tokensUsed: 80, model: "m" },
workingProviderType: "anthropic",
totalTokens: 80,
rounds: 1,
toolTrace: [],
toolResults: [],
}));
await post(chatReq({ conversationId: "c1", message: "how is my RHR?" }));
await sse.done;
expect(assistantContent()).toBe(prose);
expect(assistantProvenance()?.unverifiedFigures).toBeUndefined();
});
});
@@ -0,0 +1,125 @@
/**
* CHARACTERISATION — which tool results reach `toolResults`, and why it matters.
*
* `toolResults` is not just a trace. The chat route derives
* `activatingPayloads` from it, and the numeric grounding guard runs ONLY when
* that array is non-empty (`insights/chat/route.ts`, the
* `activatingPayloads.length > 0` gate). So whatever the loop drops here, the
* guard never sees.
*
* Two shapes of empty read, from `availability.ts` `resolveEmptyRead`:
* - nothing recorded at all -> `{ present: false, reason: "none" }`, no `available`
* - rows exist out of window -> `{ present: false, reason: "unavailable_in_scope", available }`
*
* Only the second survives `loop.ts`'s push condition. The first is dropped,
* which disarms the grounding guard for that turn. This test pins the drop so
* the consequence is visible at the seam where it happens.
*/
import { describe, expect, it, vi, beforeEach } from "vitest";
import type { CoachToolResult } from "@/lib/ai/coach/tools/executor";
const executeCoachTool = vi.fn<(args?: unknown) => Promise<CoachToolResult>>();
vi.mock("@/lib/ai/coach/tools/executor", () => ({
executeCoachTool: (args: unknown) => executeCoachTool(args),
}));
const runRawCompletionWithFallback = vi.fn();
vi.mock("@/lib/ai/provider-runner", () => ({
runRawCompletionWithFallback: (args: unknown) =>
runRawCompletionWithFallback(args),
}));
import { runCoachToolLoop } from "@/lib/ai/coach/tools/loop";
import { COACH_TOOL_DEFS } from "@/lib/ai/coach/tools/definitions";
const baseArgs = {
userId: "u1",
providers: [],
system: "sys",
messages: [{ role: "user" as const, content: "how did I sleep?" }],
tools: COACH_TOOL_DEFS,
};
describe("coach tool loop — which results reach the grounding guard", () => {
beforeEach(() => {
executeCoachTool.mockReset();
runRawCompletionWithFallback.mockReset();
});
it("a pure miss (present:false, no available) is dropped from toolResults", async () => {
executeCoachTool.mockResolvedValue({ present: false, reason: "none" });
runRawCompletionWithFallback
.mockResolvedValueOnce({
result: {
content: "",
tokensUsed: 30,
model: "mock",
providerType: "anthropic" as const,
toolCalls: [
{ id: "t1", name: "get_sleep", arguments: JSON.stringify({}) },
],
finishReason: "tool_calls",
},
workingProvider: { providerType: "anthropic" },
fallbackHops: [],
})
.mockResolvedValueOnce({
result: {
content: "You averaged 7 h 12 min of sleep.",
tokensUsed: 20,
model: "mock",
providerType: "anthropic" as const,
finishReason: "stop",
},
workingProvider: { providerType: "anthropic" },
fallbackHops: [],
});
const loop = await runCoachToolLoop(baseArgs);
// The trace records that the tool ran and found nothing…
expect(loop.toolTrace).toEqual([{ name: "get_sleep", present: false }]);
// …but the payload set the route grades against is empty, so the route's
// `activatingPayloads.length > 0` gate is false and the numeric grounding
// guard never runs for this turn.
expect(loop.toolResults).toEqual([]);
});
it("an out-of-window miss WITH `available` is kept, so the guard stays armed", async () => {
executeCoachTool.mockResolvedValue({
present: false,
reason: "unavailable_in_scope",
available: { count: 12, reachableWithWindow: "lastYear" },
} as unknown as CoachToolResult);
runRawCompletionWithFallback
.mockResolvedValueOnce({
result: {
content: "",
tokensUsed: 30,
model: "mock",
providerType: "anthropic" as const,
toolCalls: [
{ id: "t1", name: "get_sleep", arguments: JSON.stringify({}) },
],
finishReason: "tool_calls",
},
workingProvider: { providerType: "anthropic" },
fallbackHops: [],
})
.mockResolvedValueOnce({
result: {
content: "Nothing in the last 30 days.",
tokensUsed: 20,
model: "mock",
providerType: "anthropic" as const,
finishReason: "stop",
},
workingProvider: { providerType: "anthropic" },
fallbackHops: [],
});
const loop = await runCoachToolLoop(baseArgs);
expect(loop.toolResults).toHaveLength(1);
});
});
@@ -0,0 +1,133 @@
/**
* CHARACTERISATION — a rollup bucket outlives the rows it was folded from, and
* the Coach is shown the result without any marker saying so.
*
* The read-swap in `tiered-context.ts` falls back to live SQL only when a band
* is EMPTY (`if (buckets.length > 0) return buckets;`), so a bucket that is
* merely wrong is served as though it were current. The one staleness repair,
* `ensureUserRollupsFresh`, recomputes the DAY tier over the trailing 90 days
* only, and keys off `Measurement.updatedAt` — which a HARD delete does not
* bump. `src/lib/whoop/sync-body.ts` performs exactly such a hard
* `measurement.deleteMany` on a WEIGHT row with no rollup invalidation, so this
* reproduces that shape against a MONTH-band bucket.
*
* What this pins, in the snapshot the Coach narrates from:
* - `weight.timeline.coarse.monthly` still carries the deleted value;
* - `weight.aggregate.allTimeMin/Max` correctly carry only the surviving
* value, so the snapshot contradicts itself and nothing flags which half
* is right;
* - `weight.asOf.currentForTodayClaims` is true, because `asOf` is derived
* from the freshest RAW reading and knows nothing about rollup age.
*
* This asserts CURRENT behaviour. Closing the gap — comparing bucket
* `computedAt` against the type's in-window `MAX(measurement.updatedAt)`, or
* invalidating on the hard delete — will redden it, and the assertions should
* then be flipped to the honest expectation.
*/
import { beforeEach, describe, expect, it, vi } from "vitest";
process.env.ENCRYPTION_KEY ??=
"0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef";
import { getPrismaClient, truncateAllTables } from "./setup";
vi.mock("@/lib/db-compat", () => ({
ensureDbCompatibility: vi.fn().mockResolvedValue(undefined),
}));
import { buildCoachSnapshot } from "@/lib/ai/coach/snapshot";
import { recomputeUserRollups } from "@/lib/rollups/measurement-rollups";
const prisma = getPrismaClient();
const DAY = 24 * 60 * 60 * 1000;
/** The value only ever present in the rows we delete. */
const GHOST_KG = 95;
/** The value the record still holds afterwards. */
const LIVE_KG = 80;
interface WeightBlock {
aggregate: { allTimeMin: number; allTimeMax: number };
timeline: { coarse?: { monthly: Array<[string, number, number, number]> } };
asOf: { currentForTodayClaims: boolean };
}
describe("coach snapshot — a rollup bucket that outlived its rows", () => {
beforeEach(async () => {
await truncateAllTables(prisma);
});
it("serves the deleted value in the coarse tail while claiming the block is current", async () => {
const user = await prisma.user.create({
data: {
email: "stale-rollup@example.test",
username: "stale-rollup",
passwordHash: "x",
},
});
const userId = user.id;
// Rows in the MONTH band (90 days to a year), plus recent rows so the
// snapshot renders a weight block at all.
await prisma.measurement.createMany({
data: [
...Array.from({ length: 12 }, (_, i) => ({
userId,
type: "WEIGHT" as const,
value: GHOST_KG,
unit: "kg",
source: "WHOOP" as const,
externalId: `stale-${i}`,
measuredAt: new Date(Date.now() - (200 + i) * DAY),
})),
...Array.from({ length: 6 }, (_, i) => ({
userId,
type: "WEIGHT" as const,
value: LIVE_KG,
unit: "kg",
measuredAt: new Date(Date.now() - i * DAY),
})),
],
});
await recomputeUserRollups(userId);
// The hard delete `whoop/sync-body.ts` performs — no rollup invalidation,
// and no `updatedAt` bump for the freshness probe to notice.
const deleted = await prisma.measurement.deleteMany({
where: { userId, type: "WEIGHT", source: "WHOOP" },
});
expect(deleted.count).toBe(12);
const remaining = await prisma.measurement.findMany({
where: { userId, type: "WEIGHT" },
select: { value: true },
});
expect(remaining.every((r) => r.value === LIVE_KG)).toBe(true);
// The MONTH buckets are untouched by the delete.
const buckets = await prisma.measurementRollup.findMany({
where: { userId, type: "WEIGHT", granularity: "MONTH", source: "WHOOP" },
select: { mean: true, count: true },
});
expect(buckets.length).toBeGreaterThan(0);
expect(buckets.every((b) => b.mean === GHOST_KG)).toBe(true);
const snap = await buildCoachSnapshot(userId);
const weight = (snap.sections as Record<string, unknown>)
.weight as WeightBlock;
// The coarse tail the Coach narrates still reports the deleted value.
const monthly = weight.timeline.coarse?.monthly ?? [];
expect(monthly.length).toBeGreaterThan(0);
expect(monthly.every((row) => row[1] === GHOST_KG)).toBe(true);
expect(snap.snapshotJson).toContain(String(GHOST_KG));
// …while the all-time extremes, read live, know only the surviving value.
expect(weight.aggregate.allTimeMin).toBe(LIVE_KG);
expect(weight.aggregate.allTimeMax).toBe(LIVE_KG);
// …and the block is stamped current, which licenses present-tense prose.
expect(weight.asOf.currentForTodayClaims).toBe(true);
});
});