diff --git a/src/app/api/insights/chat/__tests__/route-guard-activation.test.ts b/src/app/api/insights/chat/__tests__/route-guard-activation.test.ts new file mode 100644 index 000000000..e3a7763c1 --- /dev/null +++ b/src/app/api/insights/chat/__tests__/route-guard-activation.test.ts @@ -0,0 +1,356 @@ +import { describe, it, expect, vi, beforeEach } from "vitest"; + +/** + * CHARACTERISATION — when the Coach's numeric grounding guard is armed, and + * when it is not. + * + * The guard rewrites any figure in the reply that the turn's Grounding Ledger + * cannot account for. It runs only when `activatingPayloads` is non-empty, and + * on the tool path that array comes solely from `loop.toolResults`. A tool that + * found nothing at all resolves to `{ present: false, reason: "none" }` with no + * `available` payload, and `loop.ts` drops that shape from `toolResults` + * (pinned in `tools-loop-miss-payloads.test.ts`). The no-tools snapshot + * fallback is populated only in the non-tool-mode branch, so on the tool path + * there is nothing to fall back to. + * + * The consequence, pinned below: on a tool-mode turn where every tool missed — + * i.e. exactly when the user has no data for what they asked about — a figure + * the model invented is streamed and persisted verbatim, and the reply carries + * no `unverifiedFigures` marker, so it reads to the user as fully checked. + * + * These tests assert the CURRENT behaviour, not the desired one. If the + * activation gate is widened so the guard covers the all-missed turn, they will + * fail — that failure is the fix landing, and the assertions should be flipped + * to the safe expectation at that point. + * + * The first test is the control: with one present tool result the guard is + * armed and the same invented figure is stripped. + */ + +const SNAPSHOT_JSON = '{"bp":{"aggregate":{"mean":128}}}'; + +vi.mock("@/lib/api-handler", () => ({ + apiHandler: unknown>(fn: T) => fn, + requireAuth: vi.fn(async () => ({ user: { id: "u1", locale: "en" } })), + HttpError: class HttpError extends Error { + status: number; + constructor(status: number, message: string) { + super(message); + this.status = status; + } + }, +})); +vi.mock("@/lib/modules/gate", () => ({ + requireModuleEnabled: vi.fn(async () => ({ enabled: true })), + isModuleEnabled: vi.fn(async () => true), +})); +vi.mock("@/lib/feature-flags", () => ({ + requireAssistantSurface: vi.fn(async () => undefined), +})); +vi.mock("@/lib/api-response", () => ({ + apiError: (error: string, status: number) => ({ data: null, error, status }), + apiSuccess: (data: unknown) => ({ data, error: null, status: 200 }), +})); +vi.mock("@/lib/logging/context", () => ({ + annotate: vi.fn(), + getEvent: vi.fn(), +})); +vi.mock("@/lib/logging/redact", () => ({ + redactSecrets: (s: string) => s, + redactOptional: (s: unknown) => s, +})); +vi.mock("@/lib/auth/audit", () => ({ auditLog: vi.fn() })); +vi.mock("@/lib/db", () => ({ + prisma: { + user: { findUnique: vi.fn(async () => ({ coachPrefsJson: null })) }, + coachConversation: { findFirst: vi.fn(async () => ({ id: "c1" })) }, + }, +})); + +const { checkRateLimit } = vi.hoisted(() => ({ + checkRateLimit: vi.fn(async () => ({ allowed: true })), +})); +vi.mock("@/lib/rate-limit", () => ({ checkRateLimit })); +vi.mock("@/lib/i18n/server-locale", () => ({ + resolveServerLocale: vi.fn(async () => "en"), +})); + +const { runStreamingRawCompletionWithFallback } = vi.hoisted(() => ({ + runStreamingRawCompletionWithFallback: vi.fn(), +})); +vi.mock("@/lib/ai/provider-runner", () => ({ + AllProvidersFailedError: class extends Error {}, + runStreamingRawCompletionWithFallback, +})); + +const { resolveProviderChain } = vi.hoisted(() => ({ + resolveProviderChain: vi.fn(), +})); +vi.mock("@/lib/ai/provider", () => ({ + resolveProviderChain, + resolveProvider: vi.fn(), +})); + +const { assertConsentForChain } = vi.hoisted(() => ({ + assertConsentForChain: vi.fn(async () => undefined), +})); +vi.mock("@/lib/ai/consent-guard", () => ({ assertConsentForChain })); +vi.mock("@/lib/ai/prompts/insight-generator", () => ({ PROMPT_VERSION: "x" })); +vi.mock("@/lib/ai/ai-budgets", () => ({ + AI_BUDGETS: { coach: { maxTokens: 1500, temperature: 0.4 } }, +})); + +vi.mock("@/lib/ai/coach/types", async () => { + const actual = await vi.importActual( + "@/lib/ai/coach/types", + ); + return actual; +}); + +const { fetchConversationWithMessages } = vi.hoisted(() => ({ + fetchConversationWithMessages: vi.fn(), +})); +vi.mock("@/lib/ai/coach/persistence", () => ({ + appendMessage: vi.fn(async () => ({ id: "m1" })), + createConversation: vi.fn(async () => ({ id: "c1" })), + fetchConversationWithMessages, + listConversations: vi.fn(), +})); +vi.mock("@/lib/ai/coach/coach-memory-shared", () => ({ + enqueueCoachMemoryRefresh: vi.fn(), +})); +vi.mock("@/lib/ai/coach/facts", () => ({ + storeDeterministicFacts: vi.fn(async () => undefined), +})); + +// Guard II — the schedule read. Empty here; the schedule-gated dose rule is +// unit-tested in the outbound-screen suite. +vi.mock("@/lib/medications/scheduled-doses", () => ({ + getScheduledDoseValues: vi.fn(async () => []), +})); + +const { reserveBudget, reconcileSpend } = vi.hoisted(() => ({ + reserveBudget: vi.fn(async () => ({ allowed: true, reserved: 3000 })), + reconcileSpend: vi.fn(async () => undefined), +})); +vi.mock("@/lib/ai/coach/budget", () => ({ + buildDateKey: vi.fn(() => "2026-07-23"), + reserveBudget, + reconcileSpend, + resolveDailyCap: vi.fn(() => 2_000_000), +})); + +const { detectRefusal } = vi.hoisted(() => ({ + detectRefusal: vi.fn(() => ({ refuse: false })), +})); +vi.mock("@/lib/ai/coach/refusal", () => ({ detectRefusal })); +vi.mock("@/lib/ai/coach/outbound-guard", () => ({ + screenCoachReply: vi.fn(() => ({ block: false })), + coachOutboundFallback: vi.fn(() => "fallback"), +})); +vi.mock("@/lib/ai/coach/system-prompt", () => ({ + getCoachSystemPrompt: vi.fn(() => "SYSTEM"), +})); +vi.mock("@/lib/ai/coach/about-me", () => ({ + getSelfContextTextForUser: vi.fn(async () => null), +})); +vi.mock("@/lib/ai/coach/snapshot", () => ({ + buildCoachSnapshot: vi.fn(async () => ({ + snapshotJson: SNAPSHOT_JSON, + sections: { bloodPressure: { aggregate: { mean: 128 } } }, + provenance: { windows: ["last30days"], metrics: ["bp"] }, + referenceGrounding: "REFERENCE RANGES", + })), +})); +vi.mock("@/lib/workouts/hr-series", () => ({ + buildWorkoutHrSeries: vi.fn(async () => null), +})); +vi.mock("@/lib/workouts/zones", () => ({ + computeZones: vi.fn(() => null), + hrMaxFromAge: vi.fn(() => 185), + parseWhoopZoneDurations: vi.fn(() => null), +})); +vi.mock("@/lib/workouts/sport-context", () => ({ + buildSportContext: vi.fn(async () => null), +})); + +const { + buildCoachDataInventory, + renderDataInventory, + renderFocusHint, + runCoachToolLoop, +} = vi.hoisted(() => ({ + buildCoachDataInventory: vi.fn(async () => ({ + entries: [], + restMode: false, + cycleEnabled: false, + window: "last30days", + probeScope: { sources: ["bp"], window: "last30days" }, + })), + renderDataInventory: vi.fn(() => "DATA INVENTORY\n- blood pressure: present"), + renderFocusHint: vi.fn(() => ""), + runCoachToolLoop: vi.fn(), +})); +vi.mock("@/lib/ai/coach/tools", () => ({ + COACH_TOOL_DEFS: [{ name: "get_metric_series" }], + MAX_ROUNDS: 3, + buildCoachDataInventory, + renderDataInventory, + renderFocusHint, + buildToolModeAddendum: vi.fn(() => "TOOL ADDENDUM"), + runCoachToolLoop, +})); + +const { parseKeyValuesSentinel } = vi.hoisted(() => ({ + parseKeyValuesSentinel: vi.fn(), +})); +vi.mock("@/lib/ai/coach/keyvalues", () => ({ parseKeyValuesSentinel })); +const { parseSuggestReminder } = vi.hoisted(() => ({ + parseSuggestReminder: vi.fn(), +})); +vi.mock("@/lib/ai/coach/suggest-reminder", () => ({ parseSuggestReminder })); +vi.mock("@/lib/ai/coach/suggest-gate", () => ({ gateSuggestion: vi.fn() })); +vi.mock("@/lib/validations/coach-prefs", () => ({ + parseCoachPrefs: vi.fn(() => ({ defaultWindow: undefined })), + DEFAULT_REMINDER_SUGGESTION_PREFS: {}, +})); + +const { appendMessage } = await import("@/lib/ai/coach/persistence"); + +const sse = vi.hoisted(() => ({ done: Promise.resolve() as Promise })); +vi.mock("@/lib/sse/create-stream", () => ({ + createSseStream: ( + producer: (c: { + signal: { aborted: boolean }; + enqueue: () => void; + }) => void | Promise, + ) => { + sse.done = Promise.resolve( + producer({ signal: { aborted: false }, enqueue: () => {} }), + ); + return new ReadableStream(); + }, +})); + +import { POST } from "../route"; + +const post = POST as unknown as (req: Request) => Promise; + +function chatReq(body: Record): Request { + return new Request("http://localhost/api/insights/chat", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify(body), + }); +} + +/** Force the model's assembled prose through the sentinel/suggest parsers. */ +function stubReply(prose: string, toolResults: unknown[]): void { + parseKeyValuesSentinel.mockReturnValue({ + prose, + keyValues: [], + malformed: false, + malformedEntries: [], + }); + parseSuggestReminder.mockReturnValue({ prose }); + runCoachToolLoop.mockImplementation(async () => ({ + result: { content: prose, tokensUsed: 80, model: "m" }, + workingProviderType: "anthropic", + totalTokens: 80, + rounds: 1, + toolTrace: [{ name: "get_metric_series", present: true }], + toolResults, + })); +} + +function assistantContent(): string { + const calls = (appendMessage as ReturnType).mock.calls; + const assistant = calls.find( + (c) => (c[0] as { role: string }).role === "assistant", + ); + return (assistant?.[0] as { content: string }).content; +} + +function assistantProvenance(): Record | null | undefined { + const calls = (appendMessage as ReturnType).mock.calls; + const assistant = calls.find( + (c) => (c[0] as { role: string }).role === "assistant", + ); + return ( + assistant?.[0] as { + metricSource?: Record | null; + } + ).metricSource; +} + +describe("coach chat — grounding-guard activation", () => { + beforeEach(() => { + vi.clearAllMocks(); + reserveBudget.mockResolvedValue({ allowed: true, reserved: 3000 }); + detectRefusal.mockReturnValue({ refuse: false }); + checkRateLimit.mockResolvedValue({ allowed: true }); + assertConsentForChain.mockResolvedValue(undefined); + resolveProviderChain.mockResolvedValue([ + { providerType: "anthropic", instance: {} }, + ]); + fetchConversationWithMessages.mockResolvedValue({ + id: "c1", + attachmentCount: 0, + summary: null, + messages: [], + }); + }); + + it("CONTROL — one present tool result arms the guard: the invented figure is stripped", async () => { + stubReply("Your sleep averaged 432 minutes last week.", [ + { present: true, data: { metric: "bp", aggregate: { avgSys30: 128 } } }, + ]); + + await post(chatReq({ conversationId: "c1", message: "how did I sleep?" })); + await sse.done; + + expect(assistantContent()).not.toContain("432"); + expect(assistantContent()).toContain("[…]"); + expect(assistantProvenance()?.unverifiedFigures).toBe(1); + }); + + it("every tool missed: the guard stays dormant and the invented figure survives", async () => { + // A pure miss never reaches `toolResults` (see tools-loop-miss-payloads), + // so this is what the route sees when the record holds nothing. + stubReply("Your sleep averaged 432 minutes last week.", []); + + await post(chatReq({ conversationId: "c1", message: "how did I sleep?" })); + await sse.done; + + expect(assistantContent()).toBe( + "Your sleep averaged 432 minutes last week.", + ); + // No withheld-figure notice either, so the reply reads as verified. + expect(assistantProvenance()?.unverifiedFigures).toBeUndefined(); + }); + + it("the model called no tool at all: the guard stays dormant", async () => { + const prose = "Your resting heart rate has been averaging 58 bpm."; + parseKeyValuesSentinel.mockReturnValue({ + prose, + keyValues: [], + malformed: false, + malformedEntries: [], + }); + parseSuggestReminder.mockReturnValue({ prose }); + runCoachToolLoop.mockImplementation(async () => ({ + result: { content: prose, tokensUsed: 80, model: "m" }, + workingProviderType: "anthropic", + totalTokens: 80, + rounds: 1, + toolTrace: [], + toolResults: [], + })); + + await post(chatReq({ conversationId: "c1", message: "how is my RHR?" })); + await sse.done; + + expect(assistantContent()).toBe(prose); + expect(assistantProvenance()?.unverifiedFigures).toBeUndefined(); + }); +}); diff --git a/src/lib/ai/coach/__tests__/tools-loop-miss-payloads.test.ts b/src/lib/ai/coach/__tests__/tools-loop-miss-payloads.test.ts new file mode 100644 index 000000000..c19d6e9b6 --- /dev/null +++ b/src/lib/ai/coach/__tests__/tools-loop-miss-payloads.test.ts @@ -0,0 +1,125 @@ +/** + * CHARACTERISATION — which tool results reach `toolResults`, and why it matters. + * + * `toolResults` is not just a trace. The chat route derives + * `activatingPayloads` from it, and the numeric grounding guard runs ONLY when + * that array is non-empty (`insights/chat/route.ts`, the + * `activatingPayloads.length > 0` gate). So whatever the loop drops here, the + * guard never sees. + * + * Two shapes of empty read, from `availability.ts` `resolveEmptyRead`: + * - nothing recorded at all -> `{ present: false, reason: "none" }`, no `available` + * - rows exist out of window -> `{ present: false, reason: "unavailable_in_scope", available }` + * + * Only the second survives `loop.ts`'s push condition. The first is dropped, + * which disarms the grounding guard for that turn. This test pins the drop so + * the consequence is visible at the seam where it happens. + */ +import { describe, expect, it, vi, beforeEach } from "vitest"; + +import type { CoachToolResult } from "@/lib/ai/coach/tools/executor"; + +const executeCoachTool = vi.fn<(args?: unknown) => Promise>(); +vi.mock("@/lib/ai/coach/tools/executor", () => ({ + executeCoachTool: (args: unknown) => executeCoachTool(args), +})); + +const runRawCompletionWithFallback = vi.fn(); +vi.mock("@/lib/ai/provider-runner", () => ({ + runRawCompletionWithFallback: (args: unknown) => + runRawCompletionWithFallback(args), +})); + +import { runCoachToolLoop } from "@/lib/ai/coach/tools/loop"; +import { COACH_TOOL_DEFS } from "@/lib/ai/coach/tools/definitions"; + +const baseArgs = { + userId: "u1", + providers: [], + system: "sys", + messages: [{ role: "user" as const, content: "how did I sleep?" }], + tools: COACH_TOOL_DEFS, +}; + +describe("coach tool loop — which results reach the grounding guard", () => { + beforeEach(() => { + executeCoachTool.mockReset(); + runRawCompletionWithFallback.mockReset(); + }); + + it("a pure miss (present:false, no available) is dropped from toolResults", async () => { + executeCoachTool.mockResolvedValue({ present: false, reason: "none" }); + runRawCompletionWithFallback + .mockResolvedValueOnce({ + result: { + content: "", + tokensUsed: 30, + model: "mock", + providerType: "anthropic" as const, + toolCalls: [ + { id: "t1", name: "get_sleep", arguments: JSON.stringify({}) }, + ], + finishReason: "tool_calls", + }, + workingProvider: { providerType: "anthropic" }, + fallbackHops: [], + }) + .mockResolvedValueOnce({ + result: { + content: "You averaged 7 h 12 min of sleep.", + tokensUsed: 20, + model: "mock", + providerType: "anthropic" as const, + finishReason: "stop", + }, + workingProvider: { providerType: "anthropic" }, + fallbackHops: [], + }); + + const loop = await runCoachToolLoop(baseArgs); + + // The trace records that the tool ran and found nothing… + expect(loop.toolTrace).toEqual([{ name: "get_sleep", present: false }]); + // …but the payload set the route grades against is empty, so the route's + // `activatingPayloads.length > 0` gate is false and the numeric grounding + // guard never runs for this turn. + expect(loop.toolResults).toEqual([]); + }); + + it("an out-of-window miss WITH `available` is kept, so the guard stays armed", async () => { + executeCoachTool.mockResolvedValue({ + present: false, + reason: "unavailable_in_scope", + available: { count: 12, reachableWithWindow: "lastYear" }, + } as unknown as CoachToolResult); + runRawCompletionWithFallback + .mockResolvedValueOnce({ + result: { + content: "", + tokensUsed: 30, + model: "mock", + providerType: "anthropic" as const, + toolCalls: [ + { id: "t1", name: "get_sleep", arguments: JSON.stringify({}) }, + ], + finishReason: "tool_calls", + }, + workingProvider: { providerType: "anthropic" }, + fallbackHops: [], + }) + .mockResolvedValueOnce({ + result: { + content: "Nothing in the last 30 days.", + tokensUsed: 20, + model: "mock", + providerType: "anthropic" as const, + finishReason: "stop", + }, + workingProvider: { providerType: "anthropic" }, + fallbackHops: [], + }); + + const loop = await runCoachToolLoop(baseArgs); + expect(loop.toolResults).toHaveLength(1); + }); +}); diff --git a/tests/integration/coach-stale-rollup-tail.test.ts b/tests/integration/coach-stale-rollup-tail.test.ts new file mode 100644 index 000000000..c49609ee1 --- /dev/null +++ b/tests/integration/coach-stale-rollup-tail.test.ts @@ -0,0 +1,133 @@ +/** + * CHARACTERISATION — a rollup bucket outlives the rows it was folded from, and + * the Coach is shown the result without any marker saying so. + * + * The read-swap in `tiered-context.ts` falls back to live SQL only when a band + * is EMPTY (`if (buckets.length > 0) return buckets;`), so a bucket that is + * merely wrong is served as though it were current. The one staleness repair, + * `ensureUserRollupsFresh`, recomputes the DAY tier over the trailing 90 days + * only, and keys off `Measurement.updatedAt` — which a HARD delete does not + * bump. `src/lib/whoop/sync-body.ts` performs exactly such a hard + * `measurement.deleteMany` on a WEIGHT row with no rollup invalidation, so this + * reproduces that shape against a MONTH-band bucket. + * + * What this pins, in the snapshot the Coach narrates from: + * - `weight.timeline.coarse.monthly` still carries the deleted value; + * - `weight.aggregate.allTimeMin/Max` correctly carry only the surviving + * value, so the snapshot contradicts itself and nothing flags which half + * is right; + * - `weight.asOf.currentForTodayClaims` is true, because `asOf` is derived + * from the freshest RAW reading and knows nothing about rollup age. + * + * This asserts CURRENT behaviour. Closing the gap — comparing bucket + * `computedAt` against the type's in-window `MAX(measurement.updatedAt)`, or + * invalidating on the hard delete — will redden it, and the assertions should + * then be flipped to the honest expectation. + */ +import { beforeEach, describe, expect, it, vi } from "vitest"; + +process.env.ENCRYPTION_KEY ??= + "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"; + +import { getPrismaClient, truncateAllTables } from "./setup"; + +vi.mock("@/lib/db-compat", () => ({ + ensureDbCompatibility: vi.fn().mockResolvedValue(undefined), +})); + +import { buildCoachSnapshot } from "@/lib/ai/coach/snapshot"; +import { recomputeUserRollups } from "@/lib/rollups/measurement-rollups"; + +const prisma = getPrismaClient(); +const DAY = 24 * 60 * 60 * 1000; + +/** The value only ever present in the rows we delete. */ +const GHOST_KG = 95; +/** The value the record still holds afterwards. */ +const LIVE_KG = 80; + +interface WeightBlock { + aggregate: { allTimeMin: number; allTimeMax: number }; + timeline: { coarse?: { monthly: Array<[string, number, number, number]> } }; + asOf: { currentForTodayClaims: boolean }; +} + +describe("coach snapshot — a rollup bucket that outlived its rows", () => { + beforeEach(async () => { + await truncateAllTables(prisma); + }); + + it("serves the deleted value in the coarse tail while claiming the block is current", async () => { + const user = await prisma.user.create({ + data: { + email: "stale-rollup@example.test", + username: "stale-rollup", + passwordHash: "x", + }, + }); + const userId = user.id; + + // Rows in the MONTH band (90 days to a year), plus recent rows so the + // snapshot renders a weight block at all. + await prisma.measurement.createMany({ + data: [ + ...Array.from({ length: 12 }, (_, i) => ({ + userId, + type: "WEIGHT" as const, + value: GHOST_KG, + unit: "kg", + source: "WHOOP" as const, + externalId: `stale-${i}`, + measuredAt: new Date(Date.now() - (200 + i) * DAY), + })), + ...Array.from({ length: 6 }, (_, i) => ({ + userId, + type: "WEIGHT" as const, + value: LIVE_KG, + unit: "kg", + measuredAt: new Date(Date.now() - i * DAY), + })), + ], + }); + + await recomputeUserRollups(userId); + + // The hard delete `whoop/sync-body.ts` performs — no rollup invalidation, + // and no `updatedAt` bump for the freshness probe to notice. + const deleted = await prisma.measurement.deleteMany({ + where: { userId, type: "WEIGHT", source: "WHOOP" }, + }); + expect(deleted.count).toBe(12); + + const remaining = await prisma.measurement.findMany({ + where: { userId, type: "WEIGHT" }, + select: { value: true }, + }); + expect(remaining.every((r) => r.value === LIVE_KG)).toBe(true); + + // The MONTH buckets are untouched by the delete. + const buckets = await prisma.measurementRollup.findMany({ + where: { userId, type: "WEIGHT", granularity: "MONTH", source: "WHOOP" }, + select: { mean: true, count: true }, + }); + expect(buckets.length).toBeGreaterThan(0); + expect(buckets.every((b) => b.mean === GHOST_KG)).toBe(true); + + const snap = await buildCoachSnapshot(userId); + const weight = (snap.sections as Record) + .weight as WeightBlock; + + // The coarse tail the Coach narrates still reports the deleted value. + const monthly = weight.timeline.coarse?.monthly ?? []; + expect(monthly.length).toBeGreaterThan(0); + expect(monthly.every((row) => row[1] === GHOST_KG)).toBe(true); + expect(snap.snapshotJson).toContain(String(GHOST_KG)); + + // …while the all-time extremes, read live, know only the surviving value. + expect(weight.aggregate.allTimeMin).toBe(LIVE_KG); + expect(weight.aggregate.allTimeMax).toBe(LIVE_KG); + + // …and the block is stamped current, which licenses present-tense prose. + expect(weight.asOf.currentForTodayClaims).toBe(true); + }); +});