diff --git a/CHANGELOG.md b/CHANGELOG.md index 7d9cb3b57..287df926e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,41 @@ # Changelog +## [1.41.2] — 2026-10-07 + +More room for a Coach turn, prompt caching that holds across its rounds, and +MCP tool results that match their schemas. + +### Changed + +- **Turn budget.** A turn on the person's own provider gets 165k tokens, + 200 s and 16 rounds (was 120k, 150 s, 12); a turn on the operator's budget + 55k tokens, 120 s and 8 rounds (was 40k, 90 s, 6). The final answer's + reserve keeps its share of each (1,650 tokens), the concurrent-turn slot + follows the longer wall time, and a turn shows up to 64 steps. + +### Fixed + +- **Cached input in the turn budget.** Codex requests carry a stable + `prompt_cache_key` and session id per conversation, as the official client + does, so later rounds of a turn read the prefix from the provider's cache + instead of paying for it again; `api.openai.com` gets the same key, gateways + do not. Anthropic usage now counts cache reads and cache writes in the + turn's total (they are reported beside `input_tokens`), so a cached round is + weighted at a tenth instead of almost nothing and a cache write in full. + Local and OpenAI-compatible servers that report + `prompt_tokens_details.cached_tokens` have it counted. +- **MCP output schemas (#1170).** `list_metrics` declares the `availability` + its rows already returned, the Coach-backed reads declare `searchedWindow` + and `available`, and `get_intraday_pulse` declares each bucket's `min` and + `max`. The advertised + schemas close every object, so a strict client refused these results. Every + tool's real output is now validated against its advertised schema under a + strict 2020-12 validator. Thanks to @RyRyMason for #1170. +- **Deleted readings in MCP discovery.** The `list_metrics` discovery rows and + the `search` clinical-signal probe count live rows only: a measurement type + whose readings were all deleted is no longer listed as present, and the + probes use the live-rows index instead of walking every deleted row. + ## [1.41.1] — 2026-10-07 A calmer thinking row in the Coach. diff --git a/docs/api/openapi.yaml b/docs/api/openapi.yaml index 7cb871c49..0603ce22e 100644 --- a/docs/api/openapi.yaml +++ b/docs/api/openapi.yaml @@ -1,7 +1,7 @@ openapi: 3.1.0 info: title: HealthLog API - version: 1.41.1 + version: 1.41.2 description: >- Self-hosted personal-health-tracking PWA — public API surface for the iOS native client and external ingest. @@ -52829,7 +52829,7 @@ components: id: type: string pattern: ^s[1-9]\d{0,2}$ - description: "`s1`..`s48`, unique within a turn." + description: "`s1`..`s64`, unique within a turn." tool: type: string enum: diff --git a/package.json b/package.json index 539890a1a..60195cac5 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "healthlog", - "version": "1.41.1", + "version": "1.41.2", "description": "Self-hosted personal-health-tracking PWA with Withings integration, AI insights, and doctor-report PDF export.", "license": "PolyForm-Noncommercial-1.0.0", "homepage": "https://healthlog.dev", diff --git a/public/sw.js b/public/sw.js index 2bfd0ac0d..c83b525f8 100644 --- a/public/sw.js +++ b/public/sw.js @@ -36,7 +36,7 @@ try { // v1.4.38.4 → v1.4.42. Do not hand-edit; bump `package.json` and rebuild. const CACHE_VERSION = (typeof self !== "undefined" && self.__APP_VERSION__) || - /* @sw-version-fallback */ "v1.40.6"; + /* @sw-version-fallback */ "v1.41.2"; const STATIC_CACHE = `healthlog-static-${CACHE_VERSION}`; const PAGE_CACHE = `healthlog-pages-${CACHE_VERSION}`; // v1.18.6 — read-only data cache for a curated allowlist of safe GET `/api/*` diff --git a/src/app/api/insights/chat/__tests__/__snapshots__/golden-transcripts.test.ts.snap b/src/app/api/insights/chat/__tests__/__snapshots__/golden-transcripts.test.ts.snap index b9570c8ea..7581c58cd 100644 --- a/src/app/api/insights/chat/__tests__/__snapshots__/golden-transcripts.test.ts.snap +++ b/src/app/api/insights/chat/__tests__/__snapshots__/golden-transcripts.test.ts.snap @@ -29,7 +29,7 @@ exports[`coach chat golden transcripts > 01a tool mode, two rounds, present data ], }, { - "amount": 12891, + "amount": 13341, "call": "reserveBudget", "cap": 2000000, "dateKey": "2026-09-26", @@ -78,7 +78,7 @@ exports[`coach chat golden transcripts > 01a tool mode, two rounds, present data "cached": 0, "call": "reconcileSpend", "dateKey": "2026-09-26", - "reserved": 1836, + "reserved": 2286, }, { "call": "appendMessage", @@ -278,7 +278,7 @@ exports[`coach chat golden transcripts > 01b tool mode, present data, one figure ], }, { - "amount": 12891, + "amount": 13341, "call": "reserveBudget", "cap": 2000000, "dateKey": "2026-09-26", @@ -327,7 +327,7 @@ exports[`coach chat golden transcripts > 01b tool mode, present data, one figure "cached": 0, "call": "reconcileSpend", "dateKey": "2026-09-26", - "reserved": 1836, + "reserved": 2286, }, { "call": "annotate", @@ -539,7 +539,7 @@ exports[`coach chat golden transcripts > 02 tool mode, every tool missed, fabric ], }, { - "amount": 12889, + "amount": 13339, "call": "reserveBudget", "cap": 2000000, "dateKey": "2026-09-26", @@ -588,7 +588,7 @@ exports[`coach chat golden transcripts > 02 tool mode, every tool missed, fabric "cached": 0, "call": "reconcileSpend", "dateKey": "2026-09-26", - "reserved": 1836, + "reserved": 2286, }, { "call": "annotate", @@ -1415,7 +1415,7 @@ exports[`coach chat golden transcripts > 07a all providers failed (plain) 1`] = ], }, { - "amount": 12891, + "amount": 13341, "call": "reserveBudget", "cap": 2000000, "dateKey": "2026-09-26", @@ -1453,7 +1453,7 @@ exports[`coach chat golden transcripts > 07a all providers failed (plain) 1`] = "cached": 0, "call": "reconcileSpend", "dateKey": "2026-09-26", - "reserved": 12891, + "reserved": 13341, }, { "call": "annotate", @@ -1507,7 +1507,7 @@ exports[`coach chat golden transcripts > 07b all providers failed (every hop rat ], }, { - "amount": 12891, + "amount": 13341, "call": "reserveBudget", "cap": 2000000, "dateKey": "2026-09-26", @@ -1545,7 +1545,7 @@ exports[`coach chat golden transcripts > 07b all providers failed (every hop rat "cached": 0, "call": "reconcileSpend", "dateKey": "2026-09-26", - "reserved": 12891, + "reserved": 13341, }, { "call": "annotate", @@ -1599,7 +1599,7 @@ exports[`coach chat golden transcripts > 07c all providers failed (primary crede ], }, { - "amount": 12891, + "amount": 13341, "call": "reserveBudget", "cap": 2000000, "dateKey": "2026-09-26", @@ -1637,7 +1637,7 @@ exports[`coach chat golden transcripts > 07c all providers failed (primary crede "cached": 0, "call": "reconcileSpend", "dateKey": "2026-09-26", - "reserved": 12891, + "reserved": 13341, }, { "call": "annotate", @@ -1691,7 +1691,7 @@ exports[`coach chat golden transcripts > 08 an unwrapped tagged provider error 1 ], }, { - "amount": 12891, + "amount": 13341, "call": "reserveBudget", "cap": 2000000, "dateKey": "2026-09-26", @@ -1729,7 +1729,7 @@ exports[`coach chat golden transcripts > 08 an unwrapped tagged provider error 1 "cached": 0, "call": "reconcileSpend", "dateKey": "2026-09-26", - "reserved": 12891, + "reserved": 13341, }, { "call": "annotate", @@ -1784,7 +1784,7 @@ exports[`coach chat golden transcripts > 09 an aborted request leaves a cancelle ], }, { - "amount": 12891, + "amount": 13341, "call": "reserveBudget", "cap": 2000000, "dateKey": "2026-09-26", @@ -1822,7 +1822,7 @@ exports[`coach chat golden transcripts > 09 an aborted request leaves a cancelle "cached": 0, "call": "reconcileSpend", "dateKey": "2026-09-26", - "reserved": 12891, + "reserved": 13341, }, { "call": "appendMessage", @@ -1883,7 +1883,7 @@ exports[`coach chat golden transcripts > 10 outbound block replaces the reply an ], }, { - "amount": 12892, + "amount": 13342, "call": "reserveBudget", "cap": 2000000, "dateKey": "2026-09-26", @@ -1932,7 +1932,7 @@ exports[`coach chat golden transcripts > 10 outbound block replaces the reply an "cached": 0, "call": "reconcileSpend", "dateKey": "2026-09-26", - "reserved": 1836, + "reserved": 2286, }, { "call": "annotate", @@ -2262,7 +2262,7 @@ exports[`coach chat golden transcripts > 11a KEYVALUES valid 1`] = ` ], }, { - "amount": 12891, + "amount": 13341, "call": "reserveBudget", "cap": 2000000, "dateKey": "2026-09-26", @@ -2311,7 +2311,7 @@ exports[`coach chat golden transcripts > 11a KEYVALUES valid 1`] = ` "cached": 0, "call": "reconcileSpend", "dateKey": "2026-09-26", - "reserved": 1836, + "reserved": 2286, }, { "call": "appendMessage", @@ -2515,7 +2515,7 @@ exports[`coach chat golden transcripts > 11b KEYVALUES partial 1`] = ` ], }, { - "amount": 12891, + "amount": 13341, "call": "reserveBudget", "cap": 2000000, "dateKey": "2026-09-26", @@ -2564,7 +2564,7 @@ exports[`coach chat golden transcripts > 11b KEYVALUES partial 1`] = ` "cached": 0, "call": "reconcileSpend", "dateKey": "2026-09-26", - "reserved": 1836, + "reserved": 2286, }, { "call": "annotate", @@ -2766,7 +2766,7 @@ exports[`coach chat golden transcripts > 11c KEYVALUES malformed (no closing mar ], }, { - "amount": 12891, + "amount": 13341, "call": "reserveBudget", "cap": 2000000, "dateKey": "2026-09-26", @@ -2815,7 +2815,7 @@ exports[`coach chat golden transcripts > 11c KEYVALUES malformed (no closing mar "cached": 0, "call": "reconcileSpend", "dateKey": "2026-09-26", - "reserved": 1836, + "reserved": 2286, }, { "call": "annotate", @@ -3001,7 +3001,7 @@ exports[`coach chat golden transcripts > 11d sentinel-only reply is an empty rep ], }, { - "amount": 12891, + "amount": 13341, "call": "reserveBudget", "cap": 2000000, "dateKey": "2026-09-26", @@ -3050,7 +3050,7 @@ exports[`coach chat golden transcripts > 11d sentinel-only reply is an empty rep "cached": 0, "call": "reconcileSpend", "dateKey": "2026-09-26", - "reserved": 1836, + "reserved": 2286, }, { "call": "annotate", @@ -3104,7 +3104,7 @@ exports[`coach chat golden transcripts > 11e empty provider reply 1`] = ` ], }, { - "amount": 12891, + "amount": 13341, "call": "reserveBudget", "cap": 2000000, "dateKey": "2026-09-26", @@ -3153,7 +3153,7 @@ exports[`coach chat golden transcripts > 11e empty provider reply 1`] = ` "cached": 0, "call": "reconcileSpend", "dateKey": "2026-09-26", - "reserved": 1836, + "reserved": 2286, }, ], "contentType": "text/event-stream; charset=utf-8", @@ -3198,7 +3198,7 @@ exports[`coach chat golden transcripts > 12a SUGGEST-REMINDER surfaced 1`] = ` ], }, { - "amount": 12896, + "amount": 13346, "call": "reserveBudget", "cap": 2000000, "dateKey": "2026-09-26", @@ -3247,7 +3247,7 @@ exports[`coach chat golden transcripts > 12a SUGGEST-REMINDER surfaced 1`] = ` "cached": 0, "call": "reconcileSpend", "dateKey": "2026-09-26", - "reserved": 1837, + "reserved": 2287, }, { "call": "prisma.user.update", @@ -3471,7 +3471,7 @@ exports[`coach chat golden transcripts > 12b SUGGEST-REMINDER suppressed 1`] = ` ], }, { - "amount": 12896, + "amount": 13346, "call": "reserveBudget", "cap": 2000000, "dateKey": "2026-09-26", @@ -3520,7 +3520,7 @@ exports[`coach chat golden transcripts > 12b SUGGEST-REMINDER suppressed 1`] = ` "cached": 0, "call": "reconcileSpend", "dateKey": "2026-09-26", - "reserved": 1837, + "reserved": 2287, }, { "call": "annotate", @@ -3708,7 +3708,7 @@ exports[`coach chat golden transcripts > 13 SUGGEST-ACTION 1`] = ` ], }, { - "amount": 12892, + "amount": 13342, "call": "reserveBudget", "cap": 2000000, "dateKey": "2026-09-26", @@ -3757,7 +3757,7 @@ exports[`coach chat golden transcripts > 13 SUGGEST-ACTION 1`] = ` "cached": 0, "call": "reconcileSpend", "dateKey": "2026-09-26", - "reserved": 1836, + "reserved": 2286, }, { "call": "annotate", @@ -3985,7 +3985,7 @@ exports[`coach chat golden transcripts > 14a REMEMBER valid 1`] = ` ], }, { - "amount": 12896, + "amount": 13346, "call": "reserveBudget", "cap": 2000000, "dateKey": "2026-09-26", @@ -4034,7 +4034,7 @@ exports[`coach chat golden transcripts > 14a REMEMBER valid 1`] = ` "cached": 0, "call": "reconcileSpend", "dateKey": "2026-09-26", - "reserved": 1837, + "reserved": 2287, }, { "call": "captureReminderFromSentinel", @@ -4242,7 +4242,7 @@ exports[`coach chat golden transcripts > 14b REMEMBER malformed 1`] = ` ], }, { - "amount": 12896, + "amount": 13346, "call": "reserveBudget", "cap": 2000000, "dateKey": "2026-09-26", @@ -4291,7 +4291,7 @@ exports[`coach chat golden transcripts > 14b REMEMBER malformed 1`] = ` "cached": 0, "call": "reconcileSpend", "dateKey": "2026-09-26", - "reserved": 1837, + "reserved": 2287, }, { "call": "annotate", @@ -4494,7 +4494,7 @@ exports[`coach chat golden transcripts > 15 an unknown learn link is scrubbed, a ], }, { - "amount": 12895, + "amount": 13345, "call": "reserveBudget", "cap": 2000000, "dateKey": "2026-09-26", @@ -4543,7 +4543,7 @@ exports[`coach chat golden transcripts > 15 an unknown learn link is scrubbed, a "cached": 0, "call": "reconcileSpend", "dateKey": "2026-09-26", - "reserved": 1837, + "reserved": 2287, }, { "call": "annotate", @@ -5207,7 +5207,7 @@ exports[`coach chat golden transcripts > 17a a workout-scoped first turn 1`] = ` ], }, { - "amount": 12890, + "amount": 13340, "call": "reserveBudget", "cap": 2000000, "dateKey": "2026-09-26", @@ -5256,7 +5256,7 @@ exports[`coach chat golden transcripts > 17a a workout-scoped first turn 1`] = ` "cached": 0, "call": "reconcileSpend", "dateKey": "2026-09-26", - "reserved": 1836, + "reserved": 2286, }, { "call": "appendMessage", @@ -5465,7 +5465,7 @@ exports[`coach chat golden transcripts > 17b a workout-scoped first turn with th ], }, { - "amount": 12890, + "amount": 13340, "call": "reserveBudget", "cap": 2000000, "dateKey": "2026-09-26", @@ -5514,7 +5514,7 @@ exports[`coach chat golden transcripts > 17b a workout-scoped first turn with th "cached": 0, "call": "reconcileSpend", "dateKey": "2026-09-26", - "reserved": 1836, + "reserved": 2286, }, { "call": "appendMessage", @@ -5702,7 +5702,7 @@ exports[`coach chat golden transcripts > 18 a persistence throw after the provid ], }, { - "amount": 12891, + "amount": 13341, "call": "reserveBudget", "cap": 2000000, "dateKey": "2026-09-26", @@ -5751,7 +5751,7 @@ exports[`coach chat golden transcripts > 18 a persistence throw after the provid "cached": 0, "call": "reconcileSpend", "dateKey": "2026-09-26", - "reserved": 1836, + "reserved": 2286, }, { "call": "event.setError", @@ -5817,7 +5817,7 @@ exports[`coach chat golden transcripts > 19 an existing conversation with prior ], }, { - "amount": 13053, + "amount": 13503, "call": "reserveBudget", "cap": 2000000, "dateKey": "2026-09-26", @@ -5866,7 +5866,7 @@ exports[`coach chat golden transcripts > 19 an existing conversation with prior "cached": 0, "call": "reconcileSpend", "dateKey": "2026-09-26", - "reserved": 1851, + "reserved": 2301, }, { "call": "annotate", @@ -6078,7 +6078,7 @@ exports[`coach chat golden transcripts > 20 a guided-question turn with a scope ], }, { - "amount": 12892, + "amount": 13342, "call": "reserveBudget", "cap": 2000000, "dateKey": "2026-09-26", @@ -6127,7 +6127,7 @@ exports[`coach chat golden transcripts > 20 a guided-question turn with a scope "cached": 0, "call": "reconcileSpend", "dateKey": "2026-09-26", - "reserved": 1836, + "reserved": 2286, }, { "call": "appendMessage", @@ -6295,7 +6295,7 @@ exports[`coach chat golden transcripts > 21 the German locale with a stored self ], }, { - "amount": 12917, + "amount": 13367, "call": "reserveBudget", "cap": 2000000, "dateKey": "2026-09-26", @@ -6344,7 +6344,7 @@ exports[`coach chat golden transcripts > 21 the German locale with a stored self "cached": 0, "call": "reconcileSpend", "dateKey": "2026-09-26", - "reserved": 1839, + "reserved": 2289, }, { "call": "appendMessage", @@ -6548,7 +6548,7 @@ exports[`coach chat golden transcripts > 22 tool mode with live steps: found, em ], }, { - "amount": 12891, + "amount": 13341, "call": "reserveBudget", "cap": 2000000, "dateKey": "2026-09-26", @@ -6597,7 +6597,7 @@ exports[`coach chat golden transcripts > 22 tool mode with live steps: found, em "cached": 0, "call": "reconcileSpend", "dateKey": "2026-09-26", - "reserved": 1836, + "reserved": 2286, }, { "call": "appendMessage", diff --git a/src/app/api/insights/chat/__tests__/dialog-harness.ts b/src/app/api/insights/chat/__tests__/dialog-harness.ts index 14f787db9..c9b2dc079 100644 --- a/src/app/api/insights/chat/__tests__/dialog-harness.ts +++ b/src/app/api/insights/chat/__tests__/dialog-harness.ts @@ -63,6 +63,7 @@ export interface ProviderCall { system: string; messages: Array<{ role: string; content: string }>; toolChoice: string | undefined; + cacheKey: string | undefined; toolCalls: Array<{ name: string; args: Record }>; } export const providerCalls: ProviderCall[] = []; @@ -278,6 +279,7 @@ async function runRawCompletionWithFallback(args: { system: string; messages: Array<{ role: string; content: string }>; toolChoice?: string; + cacheKey?: string; }; }) { const round = world.script.shift() ?? { text: "" }; @@ -289,6 +291,7 @@ async function runRawCompletionWithFallback(args: { content: msg.content, })), toolChoice: args.params.toolChoice, + cacheKey: args.params.cacheKey, toolCalls: calls, }); const offered = args.params.toolChoice !== "none"; diff --git a/src/app/api/insights/chat/__tests__/route-activity.test.ts b/src/app/api/insights/chat/__tests__/route-activity.test.ts index 92cb32fe5..04c689803 100644 --- a/src/app/api/insights/chat/__tests__/route-activity.test.ts +++ b/src/app/api/insights/chat/__tests__/route-activity.test.ts @@ -279,7 +279,9 @@ describe("concurrent turns", () => { expect(m.checkRateLimit).toHaveBeenCalledWith( "coach-turn-active:u1", 2, - 180_000, + // The longest turn's wall time (200 s since v1.41.2) plus 30 s for + // its reply to stream. + 230_000, ); }); }); diff --git a/src/app/api/insights/chat/route.ts b/src/app/api/insights/chat/route.ts index 971f391f8..b55b23d4f 100644 --- a/src/app/api/insights/chat/route.ts +++ b/src/app/api/insights/chat/route.ts @@ -40,6 +40,7 @@ import { runCoachTurn } from "@/lib/ai/coach/turn/pipeline"; import type { ReasoningLevel } from "@/lib/ai/reasoning/levels"; import { resolveCoachTurnReasoningLevel } from "@/lib/ai/reasoning/controls"; import { streamProviderError, streamRefusal } from "@/lib/ai/coach/turn/sse"; +import { TURN_LIMITS } from "@/lib/ai/coach/tools/turn-budget"; /** * The `coach` capability gate for a turn. Every unavailable reason throws the @@ -72,11 +73,12 @@ async function coachCapabilityRefusal(): Promise { */ const COACH_CONCURRENT_TURNS = 2; /** - * How long a turn's slot is held at most: a turn's wall time (150 s) plus - * the streaming of its reply. A slot whose release never arrived (a crashed - * process) frees itself when the window ends. + * How long a turn's slot is held at most: the longest turn's wall time + * (`TURN_LIMITS.user.wallMs`, 200 s) plus 30 s for the streaming of its + * reply. A slot whose release never arrived (a crashed process) frees itself + * when the window ends. */ -const COACH_TURN_SLOT_MS = 180_000; +const COACH_TURN_SLOT_MS = TURN_LIMITS.user.wallMs + 30_000; async function handleChatRequest(request: NextRequest): Promise { const auth = await requireAuth(); diff --git a/src/lib/ai/__tests__/cached-input-usage.test.ts b/src/lib/ai/__tests__/cached-input-usage.test.ts new file mode 100644 index 000000000..2a555aad5 --- /dev/null +++ b/src/lib/ai/__tests__/cached-input-usage.test.ts @@ -0,0 +1,368 @@ +/** + * Cached input, per provider, end to end into the Coach turn budget. + * + * The turn budget counts cached input at a tenth (`CACHED_INPUT_WEIGHT`). + * That only works when two things hold for every client: + * + * 1. `tokensUsed` is the gross count, cached input included, and + * `cachedInputTokens` is the cached share of it. The budget and the + * daily ledger both subtract the cached share from the gross. + * 2. The cached count is read from where the provider puts it: Codex and + * the Responses wire under `usage.input_tokens_details.cached_tokens`, + * Chat Completions (OpenAI, gateways, local servers that report it) + * under `usage.prompt_tokens_details.cached_tokens`, Anthropic as + * `usage.cache_read_input_tokens` beside an `input_tokens` that does + * NOT include it. + * + * Each fixture below is a usage payload in the provider's own shape for the + * same round: 20,000 tokens of input of which 18,000 came from the cache, + * and 500 of output. Every client must turn it into gross 20,500 / cached + * 18,000, which the budget weighs at 2,000 + 500 + 1,800 = 4,300. + * + * And the cache has to be hit at all: the Codex backend routes a request to + * a cache by `prompt_cache_key` (the official client sets it to its thread + * id), and `api.openai.com` documents the same field. A turn's rounds share + * one key, so round two finds round one's prefix. + */ +import { beforeEach, describe, expect, it, vi } from "vitest"; + +vi.mock("undici", async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + fetch: (input: unknown, init?: unknown) => + (globalThis.fetch as unknown as (i: unknown, n?: unknown) => unknown)( + input, + init, + ), + }; +}); +vi.mock("../local-host-allowlist", () => ({ aiEgressPolicyFor: () => ({}) })); + +import { AnthropicClient } from "../anthropic-client"; +import { CodexClient } from "../codex-client"; +import { OpenAIClient } from "../openai-client"; +import { LocalOpenAICompatibleClient } from "../local-client"; +import { singleUserTurn, type CompletionResult } from "../types"; +import { weightedRoundTokens } from "@/lib/ai/coach/tools/turn-budget"; + +const INPUT = 20_000; +const CACHED = 18_000; +const OUTPUT = 500; +const GROSS = INPUT + OUTPUT; +const WEIGHTED = INPUT - CACHED + OUTPUT + CACHED / 10; + +function expectDiscounted(result: CompletionResult) { + expect(result.tokensUsed).toBe(GROSS); + expect(result.cachedInputTokens).toBe(CACHED); + expect( + weightedRoundTokens({ + tokens: result.tokensUsed, + cachedTokens: result.cachedInputTokens, + }), + ).toBe(WEIGHTED); +} + +function sse(events: string[]): Response { + const encoder = new TextEncoder(); + return new Response( + new ReadableStream({ + start(controller) { + for (const ev of events) controller.enqueue(encoder.encode(ev)); + controller.close(); + }, + }), + { status: 200, headers: { "content-type": "text/event-stream" } }, + ); +} + +function codexStream(): Response { + return sse([ + `event: response.output_item.done\ndata: ${JSON.stringify({ + type: "response.output_item.done", + item: { + type: "message", + role: "assistant", + content: [{ type: "output_text", text: "ok" }], + }, + })}\n\n`, + `event: response.completed\ndata: ${JSON.stringify({ + type: "response.completed", + response: { + id: "resp1", + usage: { + input_tokens: INPUT, + input_tokens_details: { cached_tokens: CACHED }, + output_tokens: OUTPUT, + output_tokens_details: { reasoning_tokens: 0 }, + total_tokens: GROSS, + }, + }, + })}\n\n`, + ]); +} + +/** The Chat Completions usage block, as OpenAI and most gateways send it. */ +const chatUsage = { + prompt_tokens: INPUT, + completion_tokens: OUTPUT, + total_tokens: GROSS, + prompt_tokens_details: { cached_tokens: CACHED }, +}; + +function jsonResponse(body: unknown) { + return { + ok: true, + status: 200, + headers: { get: () => "application/json" }, + json: () => Promise.resolve(body), + text: () => Promise.resolve(JSON.stringify(body)), + }; +} + +function codexClient() { + return new CodexClient({ + accessToken: "t", + accountId: "a", + onTokenRefresh: vi + .fn() + .mockResolvedValue({ accessToken: "t", accountId: "a" }), + slugChain: ["gpt-5-codex"], + }); +} + +beforeEach(() => { + vi.restoreAllMocks(); +}); + +describe("cached input reaches the turn budget, per provider", () => { + it("codex: input_tokens_details.cached_tokens", async () => { + vi.stubGlobal("fetch", vi.fn().mockResolvedValue(codexStream())); + const result = await codexClient().generateCompletion( + singleUserTurn({ system: "s", user: "u" }), + ); + expectDiscounted(result); + }); + + it("openai (Chat Completions): prompt_tokens_details.cached_tokens", async () => { + vi.stubGlobal( + "fetch", + vi.fn().mockResolvedValue( + jsonResponse({ + choices: [{ message: { content: "ok" }, finish_reason: "stop" }], + usage: chatUsage, + }), + ), + ); + const client = new OpenAIClient({ + apiKey: "sk-test", + model: "gpt-4o-mini", + baseUrl: "https://api.openai.com/v1", + }); + expectDiscounted( + await client.generateCompletion( + singleUserTurn({ system: "s", user: "u" }), + ), + ); + }); + + it("gateway (OpenRouter and the like): the same Chat Completions block", async () => { + vi.stubGlobal( + "fetch", + vi.fn().mockResolvedValue( + jsonResponse({ + choices: [{ message: { content: "ok" }, finish_reason: "stop" }], + usage: chatUsage, + }), + ), + ); + const client = new OpenAIClient({ + apiKey: "sk-or-test", + model: "openai/gpt-4o-mini", + baseUrl: "https://openrouter.ai/api/v1", + providerType: "openai-compatible", + }); + expectDiscounted( + await client.generateCompletion( + singleUserTurn({ system: "s", user: "u" }), + ), + ); + }); + + it("anthropic: cache reads ride beside input_tokens, not inside it", async () => { + vi.stubGlobal( + "fetch", + vi.fn().mockResolvedValue( + jsonResponse({ + content: [{ type: "text", text: "ok" }], + stop_reason: "end_turn", + usage: { + // Anthropic's input_tokens counts only what came after the last + // cache breakpoint; reads and writes are reported beside it. + input_tokens: INPUT - CACHED, + cache_read_input_tokens: CACHED, + cache_creation_input_tokens: 0, + output_tokens: OUTPUT, + }, + }), + ), + ); + const client = new AnthropicClient({ + apiKey: "sk-ant-test", + model: "claude-sonnet-4-5", + }); + expectDiscounted( + await client.generateCompletion( + singleUserTurn({ system: "s", user: "u" }), + ), + ); + }); + + it("anthropic: a cache write counts in full", async () => { + vi.stubGlobal( + "fetch", + vi.fn().mockResolvedValue( + jsonResponse({ + content: [{ type: "text", text: "ok" }], + stop_reason: "end_turn", + usage: { + input_tokens: 2_000, + cache_read_input_tokens: 0, + cache_creation_input_tokens: 18_000, + output_tokens: OUTPUT, + }, + }), + ), + ); + const client = new AnthropicClient({ + apiKey: "sk-ant-test", + model: "claude-sonnet-4-5", + }); + const result = await client.generateCompletion( + singleUserTurn({ system: "s", user: "u" }), + ); + expect(result.tokensUsed).toBe(GROSS); + expect(result.cachedInputTokens ?? 0).toBe(0); + }); + + it("local: prompt_tokens_details.cached_tokens where the server reports it", async () => { + vi.stubGlobal( + "fetch", + vi.fn().mockResolvedValue( + jsonResponse({ + choices: [{ message: { content: "ok" }, finish_reason: "stop" }], + usage: chatUsage, + }), + ), + ); + const client = new LocalOpenAICompatibleClient({ + apiKey: null, + model: "llama3:8b", + baseUrl: "http://localhost:11434/v1", + }); + expectDiscounted( + await client.generateCompletion( + singleUserTurn({ system: "s", user: "u" }), + ), + ); + }); + + it("local, streamed: the usage frame's cached count", async () => { + vi.stubGlobal( + "fetch", + vi + .fn() + .mockResolvedValue( + sse([ + 'data: {"choices":[{"delta":{"content":"ok"}}]}\n\n', + `data: ${JSON.stringify({ choices: [{ delta: {}, finish_reason: "stop" }], usage: chatUsage })}\n\n`, + "data: [DONE]\n\n", + ]), + ), + ); + const client = new LocalOpenAICompatibleClient({ + apiKey: null, + model: "llama3:8b", + baseUrl: "http://localhost:11434/v1", + }); + expectDiscounted( + await client.generateCompletionStream( + singleUserTurn({ system: "s", user: "u" }), + () => {}, + ), + ); + }); +}); + +describe("a turn's rounds share one prompt cache key", () => { + it("codex sends prompt_cache_key and one session id for every round with the same key", async () => { + const fetchMock = vi + .fn() + .mockImplementation(() => Promise.resolve(codexStream())); + vi.stubGlobal("fetch", fetchMock); + const client = codexClient(); + const params = { + ...singleUserTurn({ system: "s", user: "u" }), + cacheKey: "coach-turn-key", + }; + await client.generateCompletion(params); + await client.generateCompletion(params); + + const [first, second] = fetchMock.mock.calls.map( + (call) => call[1] as { body: string; headers: Record }, + ); + expect(JSON.parse(first.body).prompt_cache_key).toBe("coach-turn-key"); + expect(JSON.parse(second.body).prompt_cache_key).toBe("coach-turn-key"); + expect(first.headers.session_id).toBe(second.headers.session_id); + expect(first.headers.session_id).toMatch( + /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/, + ); + }); + + it("codex without a key sends none and keeps a fresh session per call", async () => { + const fetchMock = vi + .fn() + .mockImplementation(() => Promise.resolve(codexStream())); + vi.stubGlobal("fetch", fetchMock); + const client = codexClient(); + await client.generateCompletion(singleUserTurn({ system: "s", user: "u" })); + await client.generateCompletion(singleUserTurn({ system: "s", user: "u" })); + const [first, second] = fetchMock.mock.calls.map( + (call) => call[1] as { body: string; headers: Record }, + ); + expect("prompt_cache_key" in JSON.parse(first.body)).toBe(false); + expect(first.headers.session_id).not.toBe(second.headers.session_id); + }); + + it("api.openai.com gets prompt_cache_key; a gateway never does", async () => { + const reply = () => + Promise.resolve( + jsonResponse({ + choices: [{ message: { content: "ok" }, finish_reason: "stop" }], + usage: chatUsage, + }), + ); + const fetchMock = vi.fn().mockImplementation(reply); + vi.stubGlobal("fetch", fetchMock); + const params = { + ...singleUserTurn({ system: "s", user: "u" }), + cacheKey: "coach-turn-key", + }; + await new OpenAIClient({ + apiKey: "sk-test", + model: "gpt-4o-mini", + baseUrl: "https://api.openai.com/v1", + }).generateCompletion(params); + await new OpenAIClient({ + apiKey: "sk-or-test", + model: "openai/gpt-4o-mini", + baseUrl: "https://openrouter.ai/api/v1", + providerType: "openai-compatible", + }).generateCompletion(params); + const [direct, gateway] = fetchMock.mock.calls.map((call) => + JSON.parse((call[1] as { body: string }).body), + ); + expect(direct.prompt_cache_key).toBe("coach-turn-key"); + expect("prompt_cache_key" in gateway).toBe(false); + }); +}); diff --git a/src/lib/ai/anthropic-client.ts b/src/lib/ai/anthropic-client.ts index 654de706b..79a681234 100644 --- a/src/lib/ai/anthropic-client.ts +++ b/src/lib/ai/anthropic-client.ts @@ -412,6 +412,7 @@ export class AnthropicClient implements AIProvider { input_tokens?: number; output_tokens?: number; cache_read_input_tokens?: number; + cache_creation_input_tokens?: number; output_tokens_details?: { thinking_tokens?: number }; }; }; @@ -487,9 +488,19 @@ export class AnthropicClient implements AIProvider { const content = wantsJson && rawText ? extractJsonObject(rawText) : (rawText ?? ""); + // v1.41.2 — the gross count includes the cached prefix. Anthropic reports + // cache reads and cache writes BESIDE `input_tokens`, which holds only + // what came after the last breakpoint; every other client reports a total + // that already contains the cached share. The turn budget and the daily + // ledger subtract `cachedInputTokens` from `tokensUsed`, so leaving the + // reads out of the gross made a mostly cached round look nearly free and + // a cache write (billed above the base rate) went uncounted. const inputTokens = json.usage?.input_tokens ?? 0; + const cacheReadTokens = json.usage?.cache_read_input_tokens ?? 0; + const cacheWriteTokens = json.usage?.cache_creation_input_tokens ?? 0; const outputTokens = json.usage?.output_tokens ?? 0; - const tokensUsed = inputTokens + outputTokens || null; + const tokensUsed = + inputTokens + cacheReadTokens + cacheWriteTokens + outputTokens || null; const cachedInputTokens = json.usage?.cache_read_input_tokens ?? null; // The presence of tool_use blocks is authoritative for the tool-loop gate: diff --git a/src/lib/ai/coach/eval/__tests__/scenarios-live.test.ts b/src/lib/ai/coach/eval/__tests__/scenarios-live.test.ts index ecf3383e1..61891818d 100644 --- a/src/lib/ai/coach/eval/__tests__/scenarios-live.test.ts +++ b/src/lib/ai/coach/eval/__tests__/scenarios-live.test.ts @@ -5,6 +5,7 @@ * chat turn's own context, the reply through the turn's parsers) and the * grade, so a live run measures the model and nothing else. */ +import { TURN_LIMITS } from "@/lib/ai/coach/tools/turn-budget"; import { afterEach, describe, expect, it, vi } from "vitest"; import type { CompletionParams, CompletionResult } from "@/lib/ai/types"; @@ -119,23 +120,27 @@ describe("runScenarioLive", () => { it("forces an answer at the round cap of the person's own plan", async () => { const call = toolRound("get_metric_table", { metric: "pulse" }); + // v1.41.2 — the cap is sixteen (was twelve); read from the limits so + // the scenario run and the turn cannot drift apart again. + const cap = TURN_LIMITS.user.maxRounds; + expect(cap).toBe(16); const provider = scripted([ - ...Array.from({ length: 11 }, () => call), + ...Array.from({ length: cap - 1 }, () => call), { content: "So far." }, ]); const observation = await runScenarioLive({ scenario: scenario("forced-final.en"), provider, }); - expect(observation.providerCalls).toBe(12); + expect(observation.providerCalls).toBe(cap); expect(provider.sent.map((p) => p.toolChoice)).toEqual([ - ...Array.from({ length: 11 }, () => "auto"), + ...Array.from({ length: cap - 1 }, () => "auto"), "none", ]); expect(observation.prose).toBe("So far."); - expect(observation.toolRounds).toBe(11); + expect(observation.toolRounds).toBe(cap - 1); // The same call every round: graded as repeats. - expect(observation.repeatedCalls).toBe(10); + expect(observation.repeatedCalls).toBe(cap - 2); }); it("ends the run on a clarifying question asked through the tool", async () => { diff --git a/src/lib/ai/coach/eval/__tests__/scenarios.test.ts b/src/lib/ai/coach/eval/__tests__/scenarios.test.ts index 9b077b2fc..bfdbc0325 100644 --- a/src/lib/ai/coach/eval/__tests__/scenarios.test.ts +++ b/src/lib/ai/coach/eval/__tests__/scenarios.test.ts @@ -63,6 +63,7 @@ import type { CoachStep, } from "@/lib/ai/coach/types"; import { COACH_FOLLOW_UP_KEYS } from "@/lib/ai/coach/dialog-keys"; +import { TURN_LIMITS } from "@/lib/ai/coach/tools/turn-budget"; import { POST } from "@/app/api/insights/chat/route"; import { @@ -130,7 +131,7 @@ const call = (name: string, args: Record = {}) => ({ }); /** Ten distinct reads: with `get_sleep` first, eleven fetching rounds. */ -const FORCED_READS: ReadonlyArray<[string, string]> = [ +const FORCED_READS: ReadonlyArray<[string, string, string?]> = [ ["pulse", "last7days"], ["pulse", "last30days"], ["pulse", "last90days"], @@ -141,6 +142,11 @@ const FORCED_READS: ReadonlyArray<[string, string]> = [ ["bp", "lastYear"], ["sleep", "last90days"], ["sleep", "lastYear"], + // v1.41.2 — four more for the sixteen-round cap. + ["sleep", "last7days"], + ["pulse", "last30days", "previous"], + ["sleep", "last30days"], + ["bp", "last30days", "previous"], ]; function script(name: string, lang: Lang): Round[] { @@ -204,12 +210,18 @@ function script(name: string, lang: Lang): Round[] { return []; case "forced-final": // v1.41 — a new read every round, never a repeat, until the round cap - // of the person's own plan (twelve, the forced answer included) makes - // the twelfth round the answer. + // of the person's own plan (sixteen since v1.41.2, the forced answer + // included) makes the last round the answer. return [ { calls: [call("get_sleep")] }, - ...FORCED_READS.map(([metric, window]) => ({ - calls: [call("get_metric_table", { metric, window })], + ...FORCED_READS.map(([metric, window, period]) => ({ + calls: [ + call("get_metric_table", { + metric, + window, + ...(period ? { period } : {}), + }), + ], })), { text: PROSE.forced[lang] }, ]; @@ -766,9 +778,14 @@ describe.each(["de", "en"] as const)("in %s", (lang) => { const scenario = byName("forced-final", lang); const { observation, frames } = await run(scenario); expect(providerCalls.map((c) => c.toolChoice)).toEqual([ - ...Array.from({ length: 11 }, () => "auto"), + ...Array.from({ length: TURN_LIMITS.user.maxRounds - 1 }, () => "auto"), "none", ]); + // v1.41.2 — every round of the turn carries the same prompt-cache key, + // so a provider that routes by key finds the prefix the last round left. + const keys = new Set(providerCalls.map((c) => c.cacheKey)); + expect(keys.size).toBe(1); + expect([...keys][0]).toMatch(/^[0-9a-f]{32}$/); const provenance = framesOf<{ metricSource: { forcedFinal?: boolean } }>( frames, "provenance", diff --git a/src/lib/ai/coach/eval/run-case.ts b/src/lib/ai/coach/eval/run-case.ts index f6d2856b0..9eecf3986 100644 --- a/src/lib/ai/coach/eval/run-case.ts +++ b/src/lib/ai/coach/eval/run-case.ts @@ -26,6 +26,7 @@ */ import type { Locale } from "@/lib/i18n/config"; import { UNBOUNDED_REACH } from "@/lib/ai/coach/history-reach"; +import { TURN_LIMITS } from "@/lib/ai/coach/tools/turn-budget"; import type { AiMessage, AiToolCall, @@ -145,7 +146,7 @@ export interface ScenarioProvider { * How many rounds a live scenario may take: the round cap of the person's * own plan (v1.41), the forced answer included. */ -const SCENARIO_MAX_ROUNDS = 12; +const SCENARIO_MAX_ROUNDS = TURN_LIMITS.user.maxRounds; function parseArgs(raw: string): Record { try { @@ -236,7 +237,7 @@ function priorResultTurns(scenario: CoachScenario) { * the chat turn's own context (system prompt, tool-mode and dialog * addenda, DATA INVENTORY, EARLIER TABLES, transcript), put to a real * model over the real tool catalogue, comparisons and the dialog tools - * for up to twelve rounds (v1.41), a question ending the run. Every call is + * for as many rounds as a turn on the person's own plan (v1.41), a question ending the run. Every call is * captured with its arguments and answered with a stand-in result; the * reply goes through the clarification and chip parsers the turn uses. * diff --git a/src/lib/ai/coach/prompt-cache-key.ts b/src/lib/ai/coach/prompt-cache-key.ts new file mode 100644 index 000000000..c70a12e96 --- /dev/null +++ b/src/lib/ai/coach/prompt-cache-key.ts @@ -0,0 +1,20 @@ +/** + * v1.41.2 — the prompt-cache key of a Coach conversation. + * + * Every round of a turn resends the same system prompt and history, and the + * next turn of the conversation resends it again with one more exchange. A + * provider that routes requests to its prompt cache by key (Codex, + * `api.openai.com`) only finds the prefix an earlier call left there when the + * calls carry the same key. One key per conversation does that; a turn with + * no conversation yet gets a key of its own, still shared by its rounds. + * + * Hashed so the provider never sees the conversation id itself. + */ +import { createHash, randomUUID } from "node:crypto"; + +export function coachPromptCacheKey(conversationId?: string | null): string { + return createHash("sha256") + .update(`coach-prompt:${conversationId || randomUUID()}`) + .digest("hex") + .slice(0, 32); +} diff --git a/src/lib/ai/coach/stream-events.ts b/src/lib/ai/coach/stream-events.ts index 5c54094bb..c0757bc43 100644 --- a/src/lib/ai/coach/stream-events.ts +++ b/src/lib/ai/coach/stream-events.ts @@ -76,7 +76,7 @@ export const coachStepSchema = z id: z .string() .regex(/^s[1-9]\d{0,2}$/) - .describe("`s1`..`s48`, unique within a turn."), + .describe("`s1`..`s64`, unique within a turn."), tool: z.enum([...COACH_TOOL_NAMES, "show_result", "snapshot"]), labelKey: z .string() diff --git a/src/lib/ai/coach/tools/__tests__/turn-budget.test.ts b/src/lib/ai/coach/tools/__tests__/turn-budget.test.ts index 2a251d442..b43779869 100644 --- a/src/lib/ai/coach/tools/__tests__/turn-budget.test.ts +++ b/src/lib/ai/coach/tools/__tests__/turn-budget.test.ts @@ -25,18 +25,40 @@ function clock(start = 0) { } describe("TURN_LIMITS", () => { + // v1.41.2 — raised by about a third from 120k / 150 s / 12 and + // 40k / 90 s / 6: a turn on a person's own plan stopped for tokens after + // five rounds while it was still reading. it("gives the person's own plan more than the operator's", () => { expect(TURN_LIMITS.user).toEqual({ - tokens: 120_000, - wallMs: 150_000, - maxRounds: 12, + tokens: 165_000, + wallMs: 200_000, + maxRounds: 16, }); expect(TURN_LIMITS.operator).toEqual({ - tokens: 40_000, - wallMs: 90_000, - maxRounds: 6, + tokens: 55_000, + wallMs: 120_000, + maxRounds: 8, }); }); + + it("keeps the final answer at the same share of each payer's tokens", () => { + expect(FINAL_ANSWER_TOKENS / TURN_LIMITS.user.tokens).toBeCloseTo(0.01); + expect(FINAL_ANSWER_TOKENS / TURN_LIMITS.operator.tokens).toBeCloseTo(0.03); + }); + + it("holds the turn's slot for at least the longest turn", async () => { + // The chat route's concurrency slot frees itself after its window; a + // window shorter than the wall time would let a third turn start while + // a long one is still running. + const { readFileSync } = await import("node:fs"); + const route = readFileSync( + `${process.cwd()}/src/app/api/insights/chat/route.ts`, + "utf8", + ); + expect(route).toMatch( + /COACH_TURN_SLOT_MS = TURN_LIMITS\.user\.wallMs \+ \d/, + ); + }); }); describe("token accounting", () => { @@ -73,9 +95,9 @@ describe("check()", () => { effort: "medium", initialInputTokens: 10_000, }); - // 34k spent; the next round with medium thinking and the final answer - // need about 8k more, past the operator's 40k. - budget.endRound({ tokens: 34_000, cachedTokens: 0, durationMs: 1_000 }); + // 49k spent; the next round with medium thinking and the final answer + // need about 8k more, past the operator's 55k. + budget.endRound({ tokens: 49_000, cachedTokens: 0, durationMs: 1_000 }); expect(budget.check()).toBe("budget"); }); @@ -103,9 +125,9 @@ describe("check()", () => { initialInputTokens: 1_000, now: c.now, }); - c.advance(30_000); + c.advance(60_000); budget.endRound({ tokens: 1_000, cachedTokens: 0, durationMs: 30_000 }); - // 30 s elapsed + two typical 30 s rounds = 90 s: not over yet. + // 60 s elapsed + two typical 30 s rounds = 120 s: not over yet. expect(budget.check()).toBeNull(); c.advance(1); budget.endRound({ tokens: 1_000, cachedTokens: 900, durationMs: 30_000 }); diff --git a/src/lib/ai/coach/tools/loop.ts b/src/lib/ai/coach/tools/loop.ts index f784a9096..ea45a44ad 100644 --- a/src/lib/ai/coach/tools/loop.ts +++ b/src/lib/ai/coach/tools/loop.ts @@ -30,6 +30,7 @@ * A clarifying question ends the turn without a prose round: the question * is the reply. */ +import { coachPromptCacheKey } from "@/lib/ai/coach/prompt-cache-key"; import type { CoachHistoryReach } from "@/lib/ai/coach/history-reach"; import { annotate } from "@/lib/logging/context"; import type { Locale } from "@/lib/i18n/config"; @@ -277,6 +278,9 @@ export async function runCoachToolLoop(args: { ), }); const progress = createProgressTracker(); + // v1.41.2 — one key for every round, so round two finds round one's prefix + // in the provider's prompt cache. + const cacheKey = coachPromptCacheKey(turn?.conversationId); const messages: AiMessage[] = [...args.messages]; let system = args.system; @@ -351,6 +355,7 @@ export async function runCoachToolLoop(args: { : {}), tools, toolChoice: isFinal ? ("none" as const) : ("auto" as const), + cacheKey, }, }); const result = fallback.result; diff --git a/src/lib/ai/coach/tools/turn-budget.ts b/src/lib/ai/coach/tools/turn-budget.ts index 3d021441c..16f8ba986 100644 --- a/src/lib/ai/coach/tools/turn-budget.ts +++ b/src/lib/ai/coach/tools/turn-budget.ts @@ -46,17 +46,26 @@ export interface TurnLimits { * Fixed per payer, not configurable: the person's own plan, key or local * model gets room for a long "why" chain; a turn the operator pays for gets * a third of the tokens and half the rounds. + * + * v1.41.2 — about a third more on both sides (was 120k / 150 s / 12 rounds + * and 40k / 90 s / 6 rounds). A real turn on a person's own plan stopped for + * tokens after five rounds while it was still reading, and the operator + * side moves with it so the split stays the same. */ export const TURN_LIMITS: Readonly> = { - user: { tokens: 120_000, wallMs: 150_000, maxRounds: 12 }, - operator: { tokens: 40_000, wallMs: 90_000, maxRounds: 6 }, + user: { tokens: 165_000, wallMs: 200_000, maxRounds: 16 }, + operator: { tokens: 55_000, wallMs: 120_000, maxRounds: 8 }, }; /** The answer budget of one round, before any thinking budget. */ export const ROUND_ANSWER_TOKENS = 600; -/** What the final answer is given on top of its input. */ -export const FINAL_ANSWER_TOKENS = 1_200; +/** + * What the final answer is given on top of its input. v1.41.2 — raised with + * the limits (was 1,200) so it keeps the same share of each payer's tokens: + * 1 % of the person's own budget, 3 % of the operator's. + */ +export const FINAL_ANSWER_TOKENS = 1_650; /** Cached input counts at this share of its size. */ export const CACHED_INPUT_WEIGHT = 0.1; diff --git a/src/lib/ai/coach/turn/__tests__/steps.test.ts b/src/lib/ai/coach/turn/__tests__/steps.test.ts index af70675a1..ccd097b50 100644 --- a/src/lib/ai/coach/turn/__tests__/steps.test.ts +++ b/src/lib/ai/coach/turn/__tests__/steps.test.ts @@ -462,7 +462,7 @@ describe("toStep — mapping", () => { parsedArgs: parsed, locale: "en", })?.id, - ).toBe("s48"); + ).toBe("s64"); expect( toStep({ call: { id: "u", name: "get_everything", arguments: "{}" }, diff --git a/src/lib/ai/coach/turn/model.ts b/src/lib/ai/coach/turn/model.ts index dc2a44810..f8c7996f7 100644 --- a/src/lib/ai/coach/turn/model.ts +++ b/src/lib/ai/coach/turn/model.ts @@ -4,6 +4,7 @@ * completion over the full snapshot. A provider failure is classified into * the structured `coach.*` code the stream answers with. */ +import { coachPromptCacheKey } from "@/lib/ai/coach/prompt-cache-key"; import { annotate } from "@/lib/logging/context"; import type { Locale } from "@/lib/i18n/config"; import { localeLanguageNames as LANGUAGE_NAMES } from "@/lib/i18n/config"; @@ -652,6 +653,7 @@ export async function runTurnModel(args: { // v1.22 (#89) — per-idle-gap timeout for the streaming local call. timeoutMs: ctx.aiResponseTimeoutMs, }), + cacheKey: coachPromptCacheKey(conversationId), ...(args.reasoning ? { reasoning: args.reasoning, diff --git a/src/lib/ai/coach/turn/steps.ts b/src/lib/ai/coach/turn/steps.ts index 2543185db..d09ce5d3f 100644 --- a/src/lib/ai/coach/turn/steps.ts +++ b/src/lib/ai/coach/turn/steps.ts @@ -40,9 +40,10 @@ import { COMPARE_SERIES_TOOL_NAME } from "@/lib/ai/coach/tools/compare-series"; /** * At most this many steps per turn; later calls run but show no step. - * v1.41 — 48: a budgeted turn runs up to twelve rounds. + * v1.41 — four per round of the longest turn: 48 for twelve rounds, 64 since + * v1.41.2 raised the cap to sixteen. */ -export const MAX_TURN_STEPS = 48; +export const MAX_TURN_STEPS = 64; const SCOPE_SOURCES: ReadonlySet = new Set( coachScopeSourceSchema.options, diff --git a/src/lib/ai/coach/types.ts b/src/lib/ai/coach/types.ts index 2db401956..686e6232f 100644 --- a/src/lib/ai/coach/types.ts +++ b/src/lib/ai/coach/types.ts @@ -498,7 +498,7 @@ export type CoachStepReason = | "invalid_arguments"; export interface CoachStep { - /** `s1`..`s48`, unique within a turn; frames upsert by id. */ + /** `s1`..`s64`, unique within a turn; frames upsert by id. */ id: string; tool: CoachToolName | "show_result" | "snapshot"; /** Closed catalog key, e.g. `coach.step.read`. */ diff --git a/src/lib/ai/codex-client.ts b/src/lib/ai/codex-client.ts index 8a4fe0fdf..2eb7f2eaf 100644 --- a/src/lib/ai/codex-client.ts +++ b/src/lib/ai/codex-client.ts @@ -1,4 +1,4 @@ -import { randomUUID } from "node:crypto"; +import { createHash, randomUUID } from "node:crypto"; import { callTimeoutMs } from "./effective-timeout"; import { safeFetch } from "@/lib/safe-fetch"; import type { @@ -431,6 +431,21 @@ export interface CodexAttemptDiagnostics { workingSlug: string | null; } +/** + * A UUID-shaped id derived from a cache key, for the session headers: the + * same key gives the same id, and the key itself never rides a header. + */ +function uuidFromKey(key: string): string { + const hex = createHash("sha256").update(`codex-session:${key}`).digest("hex"); + return [ + hex.slice(0, 8), + hex.slice(8, 12), + `4${hex.slice(13, 16)}`, + `${((parseInt(hex[16], 16) & 0x3) | 0x8).toString(16)}${hex.slice(17, 20)}`, + hex.slice(20, 32), + ].join("-"); +} + export class CodexClient implements AIProvider { readonly type = "codex" as const; /** See `AIProvider.responseTimeoutSeconds`; stamped by the resolver. */ @@ -650,8 +665,15 @@ export class CodexClient implements AIProvider { slug: string, wire: CodexReasoningWire, ): Promise { - const sessionId = randomUUID(); - const threadId = randomUUID(); + // v1.41.2 — calls that share a cache key share one session and thread + // id and send the key as `prompt_cache_key`, the way the official client + // sends its thread id (docs/codex-protocol-spec.md §3b/§3c). Before, every + // round of a Coach turn arrived as a new session with no key, so the + // backend had no reason to route it to the cache the previous round + // filled, and each round was charged its whole prefix again. + const cacheKey = params.cacheKey?.trim() || null; + const sessionId = cacheKey ? uuidFromKey(cacheKey) : randomUUID(); + const threadId = cacheKey ? sessionId : randomUUID(); // v1.18.11 — fold vision inputs (Lab-OCR) into the user turn when present. // The Codex/ChatGPT-OAuth backend accepts the `input_image` content block @@ -713,6 +735,7 @@ export class CodexClient implements AIProvider { : null, store: false, stream: true, + ...(cacheKey ? { prompt_cache_key: cacheKey } : {}), // Required field; empty array when not asking for reasoning. With // reasoning on, the encrypted items come back so the next tool // round can hand them over statelessly (`store` must stay false). diff --git a/src/lib/ai/local-client.ts b/src/lib/ai/local-client.ts index 3c3a5f184..d4ea0fbfb 100644 --- a/src/lib/ai/local-client.ts +++ b/src/lib/ai/local-client.ts @@ -11,6 +11,7 @@ import type { import { buildOpenAIMessages, mapFinishReason, + parseCachedTokens, raiseEmbeddedError, readEmbeddedError, sanitiseBodyExcerpt, @@ -411,6 +412,7 @@ export class LocalOpenAICompatibleClient implements AIProvider { }>; usage?: { total_tokens?: number; + prompt_tokens_details?: { cached_tokens?: number }; completion_tokens_details?: { reasoning_tokens?: number }; }; }; @@ -459,6 +461,11 @@ export class LocalOpenAICompatibleClient implements AIProvider { return { content, tokensUsed: json.usage?.total_tokens ?? null, + // v1.41.2 — llama.cpp, vLLM and the hosted gateways this client also + // serves report a prompt-cache hit the Chat Completions way; without it + // the Coach turn budget counted a cached prefix in full. Ollama sends + // none, which leaves it null. + cachedInputTokens: parseCachedTokens(json), model: this.config.model, providerType: "local", // v1.28.28 (#470) — surface why the model stopped so the JSON callers @@ -543,7 +550,10 @@ export class LocalOpenAICompatibleClient implements AIProvider { message?: { content?: string }; finish_reason?: string; }>; - usage?: { total_tokens?: number }; + usage?: { + total_tokens?: number; + prompt_tokens_details?: { cached_tokens?: number }; + }; } | null; // The server ignored `stream` and answered buffered — the embedded // failure can ride this body exactly as it rides the non-streaming one. @@ -577,6 +587,7 @@ export class LocalOpenAICompatibleClient implements AIProvider { return { content: buffered, tokensUsed: json?.usage?.total_tokens ?? null, + cachedInputTokens: json ? parseCachedTokens(json) : null, model: this.config.model, providerType: "local", finishReason: mapFinishReason(json?.choices?.[0]?.finish_reason), @@ -589,6 +600,7 @@ export class LocalOpenAICompatibleClient implements AIProvider { let sseBuffer = ""; let content = ""; let tokensUsed: number | null = null; + let cachedInputTokens: number | null = null; let reasoningTokens: number | null = null; let finishReason: string | undefined; @@ -679,6 +691,7 @@ export class LocalOpenAICompatibleClient implements AIProvider { }>; usage?: { total_tokens?: number; + prompt_tokens_details?: { cached_tokens?: number }; completion_tokens_details?: { reasoning_tokens?: number }; }; }; @@ -702,6 +715,8 @@ export class LocalOpenAICompatibleClient implements AIProvider { if (typeof chunk.usage?.total_tokens === "number") { tokensUsed = chunk.usage.total_tokens; } + const cached = chunk.usage?.prompt_tokens_details?.cached_tokens; + if (typeof cached === "number") cachedInputTokens = cached; const rt = chunk.usage?.completion_tokens_details?.reasoning_tokens; if (typeof rt === "number") reasoningTokens = rt; const chunkFinish = chunk.choices?.[0]?.finish_reason; @@ -761,6 +776,7 @@ export class LocalOpenAICompatibleClient implements AIProvider { return { content, tokensUsed, + cachedInputTokens, model: this.config.model, providerType: "local", finishReason: mapFinishReason(finishReason), diff --git a/src/lib/ai/openai-capabilities.ts b/src/lib/ai/openai-capabilities.ts index 75b723a8e..1766ec05d 100644 --- a/src/lib/ai/openai-capabilities.ts +++ b/src/lib/ai/openai-capabilities.ts @@ -34,7 +34,7 @@ function isModernCanonicalModel(model: string): boolean { ); } -function isCanonicalOpenAIEndpoint(baseUrl: string): boolean { +export function isCanonicalOpenAIEndpoint(baseUrl: string): boolean { try { const url = new URL(baseUrl); return ( diff --git a/src/lib/ai/openai-client.ts b/src/lib/ai/openai-client.ts index 227613d67..46c084158 100644 --- a/src/lib/ai/openai-client.ts +++ b/src/lib/ai/openai-client.ts @@ -3,6 +3,7 @@ import { callTimeoutMs } from "./effective-timeout"; import { annotate } from "@/lib/logging/context"; import type { AIProvider, CompletionParams, CompletionResult } from "./types"; import { + isCanonicalOpenAIEndpoint, selectOpenAIChatCompletionsCapabilities, type OpenAIChatCompletionsCapabilities, } from "./openai-capabilities"; @@ -208,6 +209,13 @@ export class OpenAIClient implements AIProvider { : {}), ...(tools ? { tools } : {}), ...(params.toolChoice ? { tool_choice: params.toolChoice } : {}), + // v1.41.2 — the prompt-cache routing key `api.openai.com` documents + // (https://platform.openai.com/docs/guides/prompt-caching). Only + // there: a gateway or proxy behind another base URL may refuse a + // field it does not know. + ...(params.cacheKey && isCanonicalOpenAIEndpoint(this.config.baseUrl) + ? { prompt_cache_key: params.cacheKey } + : {}), // #1126 — the gateway entry's reasoning setting; Default sends // nothing. `api.openai.com` and Codex never get it. v1.41 — a call // that carries its own resolved level (`params.reasoning`) wins. diff --git a/src/lib/ai/openai-wire.ts b/src/lib/ai/openai-wire.ts index 673c128a7..f7ebb3ff1 100644 --- a/src/lib/ai/openai-wire.ts +++ b/src/lib/ai/openai-wire.ts @@ -150,7 +150,9 @@ export function parseOpenAIToolCalls( })); } -export function parseCachedTokens(json: OpenAIResponseJson): number | null { +export function parseCachedTokens(json: { + usage?: { prompt_tokens_details?: { cached_tokens?: number } }; +}): number | null { return json.usage?.prompt_tokens_details?.cached_tokens ?? null; } diff --git a/src/lib/ai/types.ts b/src/lib/ai/types.ts index c33949131..be66d3c42 100644 --- a/src/lib/ai/types.ts +++ b/src/lib/ai/types.ts @@ -374,6 +374,16 @@ export interface CompletionParams { * client sends exactly what it sent before the field existed. */ reasoning?: { effort: ReasoningLevel; summaries: boolean }; + /** + * v1.41.2 — a stable key for calls that resend the same prompt prefix (the + * rounds of one Coach turn, the turns of one conversation). Providers that + * route a request to a prompt cache by key read it: Codex as + * `prompt_cache_key` and its session headers, `api.openai.com` as + * `prompt_cache_key`. Without it a Codex call is routed afresh every time + * and a later round pays for its whole prefix again. An opaque value: the + * caller hashes whatever it derives the key from. Absent → nothing is sent. + */ + cacheKey?: string; /** * v1.41 — live reasoning events, called as they arrive on clients that read * the provider's stream, or once after the reply on clients that do not. @@ -390,6 +400,12 @@ export interface CompletionResult { * from the provider's cache, where the provider reports it (Anthropic * `cache_read_input_tokens`, OpenAI/Codex `cached_tokens`). Null / absent * when the provider does not report it or the prefix did not hit. + * + * v1.41.2 — always a share of `tokensUsed`, never beside it: the turn + * budget and the daily ledger charge `tokensUsed - cachedInputTokens` at + * full weight and the cached share at a tenth. A client whose provider + * reports cached input outside its input count (Anthropic) adds it to + * `tokensUsed`. */ cachedInputTokens?: number | null; model: string; diff --git a/src/lib/mcp/__tests__/measurement-live-rows-guard.test.ts b/src/lib/mcp/__tests__/measurement-live-rows-guard.test.ts new file mode 100644 index 000000000..8b18efe71 --- /dev/null +++ b/src/lib/mcp/__tests__/measurement-live-rows-guard.test.ts @@ -0,0 +1,112 @@ +/** + * Guard: every measurement read on the MCP surface and in the Coach tools + * reads live rows only. + * + * A deleted measurement is a tombstone (`deletedAt` set), not a missing row. + * Two presence probes on the MCP side (`metricStatusDiscoveryRows` and the + * `search` clinical-signal probe) grouped by type without `deletedAt: null`: + * a type whose readings had all been deleted still counted as present, and + * because the live-rows partial index (`measurements_live_covering_idx`, + * `WHERE deleted_at IS NULL`) only serves a query that carries the same + * predicate, the probe walked every tombstone of the account (about a second + * against 23 ms on a large production record). + * + * The guard reads the source of both trees, finds every + * `prisma.measurement.(` call, takes its argument up to the matching + * parenthesis, and asserts the argument names `deletedAt`. Unique-key + * lookups (`findUnique`) are idempotency checks on a key a tombstone still + * holds, and are left out on purpose. Raw SQL against `measurements` must + * carry `deleted_at` the same way. + */ +import { readFileSync, readdirSync, statSync } from "node:fs"; +import { join, relative } from "node:path"; +import { describe, expect, it } from "vitest"; + +const ROOTS = ["src/lib/mcp", "src/lib/ai/coach/tools"]; +const READS = + /\bprisma\.measurement\.(findMany|findFirst|count|groupBy|aggregate)\s*\(/g; +const RAW = /\b(?:FROM|JOIN)\s+"?measurements"?\b/gi; + +function sourceFiles(dir: string): string[] { + return readdirSync(dir).flatMap((name) => { + const path = join(dir, name); + if (statSync(path).isDirectory()) { + return name === "__tests__" ? [] : sourceFiles(path); + } + return /\.tsx?$/.test(name) && !/\.test\.tsx?$/.test(name) ? [path] : []; + }); +} + +/** The text from `open` (an opening parenthesis) to its matching close. */ +function balanced(text: string, open: number): string { + let depth = 0; + for (let i = open; i < text.length; i += 1) { + if (text[i] === "(") depth += 1; + else if (text[i] === ")") { + depth -= 1; + if (depth === 0) return text.slice(open, i + 1); + } + } + return text.slice(open); +} + +interface Site { + at: string; + argument: string; +} + +function readSites(): { calls: Site[]; raw: Site[] } { + const calls: Site[] = []; + const raw: Site[] = []; + for (const root of ROOTS) { + for (const file of sourceFiles(join(process.cwd(), root))) { + const text = readFileSync(file, "utf8"); + const name = relative(process.cwd(), file); + const lineOf = (index: number) => text.slice(0, index).split("\n").length; + for (const match of text.matchAll(READS)) { + const open = (match.index ?? 0) + match[0].length - 1; + calls.push({ + at: `${name}:${lineOf(match.index ?? 0)}`, + argument: balanced(text, open), + }); + } + for (const match of text.matchAll(RAW)) { + // The statement around it: up to the closing backtick of the + // tagged template the query lives in. + const start = match.index ?? 0; + const end = text.indexOf("`", start); + raw.push({ + at: `${name}:${lineOf(start)}`, + argument: text.slice(start, end === -1 ? undefined : end), + }); + } + } + } + return { calls, raw }; +} + +describe("measurement reads on the MCP surface and in the Coach tools", () => { + const { calls, raw } = readSites(); + + it("finds the reads it is meant to check", () => { + // A matcher that matches nothing proves nothing: the two probes this + // guard was written for, the rich-read counts and the availability + // probe are all in these trees. + expect(calls.length).toBeGreaterThanOrEqual(5); + expect(raw.length).toBeGreaterThanOrEqual(1); + }); + + it("filters every Prisma read to live rows", () => { + const missing = calls + .filter((site) => !/\bdeletedAt\b/.test(site.argument)) + .map((site) => site.at); + expect(missing).toEqual([]); + }); + + it("filters every raw read to live rows", () => { + const missing = raw + .filter((site) => !/\bdeleted_at\b/.test(site.argument)) + .map((site) => site.at); + expect(missing).toEqual([]); + }); +}); diff --git a/src/lib/mcp/rich-reads.ts b/src/lib/mcp/rich-reads.ts index 2dcae91f3..a0f9efcaa 100644 --- a/src/lib/mcp/rich-reads.ts +++ b/src/lib/mcp/rich-reads.ts @@ -481,11 +481,16 @@ export async function metricStatusDiscoveryRows(userId: string): Promise< metric: string; }> > { + // v1.41.2 — live rows only. Without `deletedAt: null` a type whose rows + // were all deleted still counted as present, and the grouped count could + // not use the live-rows partial index (`measurements_live_covering_idx`), + // so it walked every tombstone on the way. const rows = await prisma.measurement.groupBy({ by: ["type"], where: { userId, type: { in: MCP_METRIC_STATUS_DISCOVERY.map((s) => s.measurementType) }, + deletedAt: null, }, _count: { _all: true }, }); diff --git a/src/lib/mcp/tools.ts b/src/lib/mcp/tools.ts index befde5e00..ad31be703 100644 --- a/src/lib/mcp/tools.ts +++ b/src/lib/mcp/tools.ts @@ -537,11 +537,15 @@ function searchAndFetchTools(): McpToolDefinition[] { ...MCP_CLINICAL_SIGNALS, ...MCP_METRIC_STATUS_DISCOVERY, ]; + // v1.41.2 — live rows only: a deleted reading is not a signal the + // record holds, and the filter lets the probe use the live-rows + // partial index instead of walking the tombstones. const discoverablePresent = await prisma.measurement.groupBy({ by: ["type"], where: { userId: ctx.userId, type: { in: discoverableSignals.map((s) => s.measurementType) }, + deletedAt: null, }, }); const presentTypes = new Set(discoverablePresent.map((r) => r.type)); @@ -815,6 +819,31 @@ const coachReadOutput: z.ZodRawShape = { reason: z.string().optional(), data: z.unknown().optional(), grounding: z.string().optional(), + // v1.41.2 — a miss describes its own scope and, when the record holds rows + // the window could not reach, what it holds (`CoachToolResult`). The + // executor has returned both since v1.37; undeclared, they made a strict + // client refuse every out-of-window read (the #1170 class). + searchedWindow: z.string().optional(), + available: z + .object({ + count: z.number(), + firstDate: z.string(), + lastDate: z.string(), + series: z + .array( + z.object({ + series: z.string(), + unit: z.string().nullable(), + count: z.number(), + mean: z.number(), + min: z.number(), + max: z.number(), + }), + ) + .optional(), + reachableWithWindow: z.string().nullable(), + }) + .optional(), }; /** @@ -895,6 +924,20 @@ const listMetricsOutput: z.ZodRawShape = { present: z.boolean(), count: z.number().optional(), metric: z.string().optional(), + // #1170 — what the record holds for a domain the window found + // nothing in (`InventoryEntry.availability`). The handler always + // returned it; the advertised schema did not declare it, and the + // output schema closes every object, so a strict client refused the + // whole result. + availability: z + .object({ + state: z.string(), + count: z.number().optional(), + firstDate: z.string().optional(), + lastDate: z.string().optional(), + reachableWithWindow: z.string().nullable(), + }) + .optional(), }), ) .optional(), @@ -1211,6 +1254,10 @@ const getIntradayPulseOutput: z.ZodRawShape = { startMinute: z.number(), mean: z.number(), count: z.number(), + // v1.41.2 — the bucket's low / high where the source carries a + // spread (`IntradayHrBucket`); returned since v1.30.7, never declared. + min: z.number().optional(), + max: z.number().optional(), }), ) .optional(), diff --git a/tests/integration/mcp-output-schema-conformance.test.ts b/tests/integration/mcp-output-schema-conformance.test.ts new file mode 100644 index 000000000..ec72c2d0a --- /dev/null +++ b/tests/integration/mcp-output-schema-conformance.test.ts @@ -0,0 +1,504 @@ +/** + * Every MCP tool's real output validates against the output schema + * `tools/list` advertises, under a strict JSON Schema 2020-12 validator. + * + * The advertised output schema closes every object (`additionalProperties: + * false`, the `io: "output"` rendering). The SDK checks `structuredContent` + * against the Zod shape on the server, and a Zod object strips a key it does + * not declare instead of refusing it, so a handler that returns a field its + * schema never declared passes every server-side check while a strict client + * refuses the whole result. That is how `list_metrics` shipped + * `availability` undeclared (#1170). + * + * This file closes the class rather than the one field: a real server over + * an in-memory transport, a real Postgres record with enough in it that the + * tools return their full shapes (present series, a history the default + * window cannot reach, labs, a medication, a workout, sleep stages, mood, + * clinical signals), and every advertised tool called with arguments that + * reach its main branches. Each `structuredContent` is validated against the + * schema exactly as it was advertised. A tool the case table does not cover + * fails the run, so a new tool cannot ship without joining it. + * + * It also pins the live-rows rule on the two presence probes: a measurement + * type whose readings were all deleted is not listed as present. + */ +import { beforeAll, describe, expect, it } from "vitest"; +import Ajv2020 from "ajv/dist/2020"; + +process.env.APP_URL ??= "http://localhost"; + +import { Client } from "@modelcontextprotocol/sdk/client/index.js"; +import { InMemoryTransport } from "@modelcontextprotocol/sdk/inMemory.js"; + +import { createMcpServer } from "@/lib/mcp/server"; +import { + MCP_CLINICAL_SIGNALS, + MCP_METRIC_STATUS_DISCOVERY, +} from "@/lib/mcp/rich-reads"; +import type { McpAuthContext } from "@/lib/mcp/auth"; + +import { getPrismaClient, truncateAllTables } from "./setup"; + +const DAY = 86_400_000; +const now = Date.now(); +const daysAgo = (n: number, hour = 8) => { + const d = new Date(now - n * DAY); + d.setUTCHours(hour, 0, 0, 0); + return d; +}; +const isoDay = (d: Date) => d.toISOString().slice(0, 10); + +/** A discovery metric with live rows, and one with only tombstones. */ +const LIVE_STATUS = MCP_METRIC_STATUS_DISCOVERY.find( + (s) => s.key === "CARDIO_RECOVERY", +)!; +const DELETED_STATUS = MCP_METRIC_STATUS_DISCOVERY.find( + (s) => s.key === "WRIST_TEMPERATURE", +)!; +const LIVE_SIGNAL = MCP_CLINICAL_SIGNALS.find((s) => s.key === "PAIN_NRS")!; +const DELETED_SIGNAL = MCP_CLINICAL_SIGNALS.find( + (s) => s.key === "GRIP_STRENGTH", +)!; + +let userId = ""; + +async function seedRecord(): Promise { + const prisma = getPrismaClient(); + await truncateAllTables(prisma); + const user = await prisma.user.create({ + data: { + username: "mcp-conformance", + email: "mcp-conformance@example.test", + timezone: "UTC", + createdAt: new Date(now - 800 * DAY), + }, + }); + const id = user.id; + + type Row = { + userId: string; + type: string; + value: number; + unit: string; + measuredAt: Date; + source?: string; + sleepStage?: string; + deletedAt?: Date; + }; + const rows: Row[] = []; + const daily = ( + type: string, + unit: string, + days: number, + value: (i: number) => number, + extra: Partial = {}, + ) => { + for (let i = 1; i <= days; i += 1) { + rows.push({ + userId: id, + type, + unit, + value: value(i), + measuredAt: daysAgo(i), + ...extra, + }); + } + }; + daily("WEIGHT", "kg", 120, (i) => 80 + (i % 7) * 0.2); + daily("BODY_FAT", "%", 60, (i) => 22 + (i % 5) * 0.1); + daily("BLOOD_PRESSURE_SYS", "mmHg", 60, (i) => 118 + (i % 9)); + daily("BLOOD_PRESSURE_DIA", "mmHg", 60, (i) => 76 + (i % 6)); + daily("RESTING_HEART_RATE", "bpm", 60, (i) => 58 + (i % 4)); + daily("HEART_RATE_VARIABILITY", "ms", 60, (i) => 45 + (i % 10)); + daily("ACTIVITY_STEPS", "count", 60, (i) => 6_000 + i * 37); + daily("BLOOD_GLUCOSE", "mg/dL", 30, (i) => 90 + (i % 20)); + daily(LIVE_STATUS.measurementType, "s", 30, (i) => 25 + (i % 3)); + daily(LIVE_SIGNAL.measurementType, "score", 20, (i) => i % 6); + // A domain the default window cannot reach: readings 200 days back only. + for (let i = 0; i < 6; i += 1) { + rows.push({ + userId: id, + type: "VO2_MAX", + unit: "mL/kg/min", + value: 41 + i * 0.3, + measuredAt: daysAgo(200 + i), + }); + } + // Pulse through yesterday, hourly, so the intraday read has a day. + for (let i = 1; i <= 14; i += 1) { + for (let h = 0; h < 24; h += 2) { + rows.push({ + userId: id, + type: "PULSE", + unit: "bpm", + value: 60 + ((i + h) % 25), + measuredAt: daysAgo(i, h), + }); + } + } + // Sleep stages per night. + for (let i = 1; i <= 30; i += 1) { + for (const [stage, minutes] of [ + ["CORE", 210], + ["DEEP", 60], + ["REM", 90], + ["AWAKE", 15], + ] as const) { + rows.push({ + userId: id, + type: "SLEEP_DURATION", + unit: "minutes", + value: minutes, + measuredAt: daysAgo(i, 6), + source: "APPLE_HEALTH", + sleepStage: stage, + }); + } + } + // Tombstones only: these two types must not count as present. + daily(DELETED_STATUS.measurementType, "°C", 40, () => 0.2, { + deletedAt: new Date(now - DAY), + }); + daily(DELETED_SIGNAL.measurementType, "kg", 40, () => 38, { + deletedAt: new Date(now - DAY), + }); + await prisma.measurement.createMany({ + data: rows.map((r) => ({ source: "MANUAL", ...r })) as never, + }); + + await prisma.labResult.createMany({ + data: [ + { + userId: id, + analyte: "LDL", + value: 3.1, + unit: "mmol/L", + takenAt: daysAgo(40), + }, + { + userId: id, + analyte: "LDL", + value: 2.8, + unit: "mmol/L", + takenAt: daysAgo(10), + }, + { + userId: id, + analyte: "HbA1c", + value: 5.4, + unit: "%", + takenAt: daysAgo(10), + }, + ], + }); + const med = await prisma.medication.create({ + data: { + userId: id, + name: "Ramipril", + dose: "5mg", + active: true, + createdAt: daysAgo(60), + schedules: { + create: { + windowStart: "08:00", + windowEnd: "09:00", + timesOfDay: ["08:00"], + scheduleType: "SCHEDULED", + }, + }, + }, + }); + for (let i = 1; i <= 14; i += 1) { + await prisma.medicationIntakeEvent.create({ + data: { + userId: id, + medicationId: med.id, + scheduledFor: daysAgo(i, 8), + takenAt: i % 5 === 0 ? null : daysAgo(i, 8), + skipped: i % 5 === 0, + source: "REMINDER", + }, + }); + } + await prisma.workout.create({ + data: { + userId: id, + sportType: "RUNNING", + startedAt: daysAgo(3, 17), + endedAt: new Date(daysAgo(3, 17).getTime() + 40 * 60_000), + durationSec: 2_400, + source: "MANUAL", + }, + }); + for (let i = 1; i <= 10; i += 1) { + await prisma.moodEntry.create({ + data: { + userId: id, + date: isoDay(daysAgo(i)), + mood: "GUT", + score: 3 + (i % 3), + moodLoggedAt: daysAgo(i, 20), + }, + }); + } + return id; +} + +function ctxFor(id: string): McpAuthContext { + return { + userId: id, + tokenId: "token-conformance", + scopes: ["health:read", "health:write"], + binding: `${id}:token-conformance`, + canRead: true, + canWrite: true, + }; +} + +async function connect(id: string) { + const server = createMcpServer(ctxFor(id)); + const client = new Client({ name: "conformance", version: "1.0.0" }); + const [clientSide, serverSide] = InMemoryTransport.createLinkedPair(); + await Promise.all([server.connect(serverSide), client.connect(clientSide)]); + return { + client, + close: async () => { + await client.close(); + await server.close(); + }, + }; +} + +/** + * The arguments each tool is called with: one entry per call, chosen to reach + * the tool's main branches. `fetch` takes its ids from `search` at run time. + */ +const CASES: Record>> = { + list_metrics: [{}], + get_metric_series: [ + { metric: "weight", window: "last90days" }, + { metric: "bp", window: "last30days" }, + { metric: "sleep", window: "last30days" }, + { metric: "vo2_max", window: "last30days" }, + ], + get_medication_compliance: [{ window: "last30days" }], + get_labs: [{}, { analyte: "LDL", history: true }, { analyte: "Ferritin" }], + get_correlations: [{}], + get_correlation: [{ metricA: "weight", metricB: "resting_hr" }], + compare_metric: [ + { metric: "weight", window: "last7days", windowB: "last30days" }, + { metric: "weight", metricB: "body_fat", window: "last30days" }, + { metric: LIVE_STATUS.key, window: "last30days" }, + ], + get_metric_baseline: [{ metric: "resting_hr" }, { metric: LIVE_SIGNAL.key }], + detect_changepoints: [{ metric: "weight", window: "last90days" }], + get_glucose_panel: [{ window: "last30days" }], + get_sleep: [{ window: "last30days" }], + get_workouts: [{ window: "last30days" }], + get_illness_recovery: [{}], + get_cycle: [{}], + get_metrics: [ + { metrics: ["weight", "bp", "steps"], window: "last30days" }, + { metrics: ["hrv", "vo2_max"] }, + ], + get_medication_schedule: [{}], + get_integration_status: [{}], + get_preventive_care: [{}], + get_nutrients: [{}, { nutrient: "water", days: 7 }], + get_intraday_pulse: [{ date: isoDay(daysAgo(1)) }, {}], + get_ecg_recordings: [{}], + get_visits: [{}, { months: 12 }], + search: [ + { query: "blood pressure" }, + { query: "LDL" }, + { query: "pain" }, + { query: "ramipril" }, + ], + fetch: [], + log_measurement: [ + { type: "WEIGHT", value: 80.4, unit: "kg", idempotencyKey: "conf-w-1" }, + { + type: "WEIGHT", + value: 80.4, + unit: "kg", + idempotencyKey: "conf-w-1", + confirm: true, + }, + ], + log_mood: [ + { score: 4, idempotencyKey: "conf-m-1" }, + { score: 4, idempotencyKey: "conf-m-1", confirm: true }, + ], + log_blood_pressure: [ + { systolic: 121, diastolic: 79, idempotencyKey: "conf-bp-1" }, + { + systolic: 121, + diastolic: 79, + idempotencyKey: "conf-bp-1", + confirm: true, + }, + ], +}; + +beforeAll(async () => { + userId = await seedRecord(); +}); + +describe("MCP tool outputs against their advertised schemas", () => { + it("every structuredContent validates under a strict 2020-12 validator", async () => { + const { client, close } = await connect(userId); + try { + const { tools } = await client.listTools(); + expect(tools.length).toBeGreaterThan(20); + const uncovered = tools + .map((t) => t.name) + .filter((name) => !(name in CASES)); + expect(uncovered, "add every new tool to CASES").toEqual([]); + + const ajv = new Ajv2020({ strict: true, allErrors: true }); + const failures: string[] = []; + let validated = 0; + // Tools that answered with data, not only a miss: a run where every + // read came back empty would validate nothing but the miss shape. + const presentReads = new Set(); + const fetchIds: string[] = []; + + const check = async ( + name: string, + schema: Record | undefined, + args: Record, + ) => { + const label = `${name}(${JSON.stringify(args)})`; + let result: Awaited>; + try { + // The SDK client validates against the advertised schema too, with + // its own validator; its refusal is a failure of the same kind. + result = await client.callTool({ name, arguments: args }); + } catch (err) { + failures.push(`${label}: ${(err as Error).message}`); + return undefined; + } + if (result.isError) { + failures.push( + `${label}: tool error ${JSON.stringify(result.content)}`, + ); + return undefined; + } + expect(schema, `${name} advertises an output schema`).toBeDefined(); + const structured = result.structuredContent; + if (!structured) { + failures.push(`${label}: no structuredContent`); + return undefined; + } + const validate = ajv.compile(schema!); + if (!validate(structured)) { + failures.push( + `${label}: ${ajv.errorsText(validate.errors, { separator: "; " })}`, + ); + } + validated += 1; + if ((structured as { present?: unknown }).present === true) { + presentReads.add(name); + } + return structured; + }; + + for (const tool of tools) { + if (tool.name === "fetch") continue; + for (const args of CASES[tool.name]) { + const out = await check( + tool.name, + tool.outputSchema as Record | undefined, + args, + ); + if (tool.name === "search" && out) { + for (const hit of (out as { results: Array<{ id: string }> }) + .results) { + fetchIds.push(hit.id); + } + } + } + } + + const fetchTool = tools.find((t) => t.name === "fetch")!; + const distinct = [...new Set(fetchIds)]; + expect(distinct.length).toBeGreaterThan(3); + for (const id of distinct) { + await check( + "fetch", + fetchTool.outputSchema as Record, + { id }, + ); + } + + expect(failures).toEqual([]); + expect(validated).toBeGreaterThan(Object.keys(CASES).length); + expect(presentReads.size).toBeGreaterThanOrEqual(12); + } finally { + await close(); + } + }); + + it("list_metrics carries availability, and it is declared", async () => { + const { client, close } = await connect(userId); + try { + const result = await client.callTool({ + name: "list_metrics", + arguments: {}, + }); + const metrics = ( + result.structuredContent as { + metrics: Array<{ present: boolean; availability?: unknown }>; + } + ).metrics; + // The seeded VO2 max history sits outside the default window: the + // row is absent but says what the record holds. Without it the + // conformance run above would not have exercised the field. + expect(metrics.some((m) => !m.present && m.availability)).toBe(true); + } finally { + await close(); + } + }); +}); + +describe("tombstones on the presence probes", () => { + it("list_metrics lists a live discovery metric and not one whose rows were all deleted", async () => { + const { client, close } = await connect(userId); + try { + const result = await client.callTool({ + name: "list_metrics", + arguments: {}, + }); + const metrics = ( + result.structuredContent as { + metrics: Array<{ metric?: string; present: boolean }>; + } + ).metrics; + const byMetric = new Map(metrics.map((m) => [m.metric, m])); + expect(byMetric.get(LIVE_STATUS.key)?.present).toBe(true); + expect(byMetric.get(DELETED_STATUS.key)?.present ?? false).toBe(false); + } finally { + await close(); + } + }); + + it("search finds a live clinical signal and not one whose rows were all deleted", async () => { + const { client, close } = await connect(userId); + try { + const ids = async (query: string) => + ( + (await client.callTool({ name: "search", arguments: { query } })) + .structuredContent as { results: Array<{ id: string }> } + ).results.map((r) => r.id); + expect(await ids("pain")).toContain(`metric:${LIVE_SIGNAL.key}`); + expect(await ids("grip strength")).not.toContain( + `metric:${DELETED_SIGNAL.key}`, + ); + expect(await ids("wrist temperature")).not.toContain( + `metric:${DELETED_STATUS.key}`, + ); + } finally { + await close(); + } + }); +});