diff --git a/src/app/api/insights/chat/__tests__/__snapshots__/golden-transcripts.test.ts.snap b/src/app/api/insights/chat/__tests__/__snapshots__/golden-transcripts.test.ts.snap index b9570c8ea..7581c58cd 100644 --- a/src/app/api/insights/chat/__tests__/__snapshots__/golden-transcripts.test.ts.snap +++ b/src/app/api/insights/chat/__tests__/__snapshots__/golden-transcripts.test.ts.snap @@ -29,7 +29,7 @@ exports[`coach chat golden transcripts > 01a tool mode, two rounds, present data ], }, { - "amount": 12891, + "amount": 13341, "call": "reserveBudget", "cap": 2000000, "dateKey": "2026-09-26", @@ -78,7 +78,7 @@ exports[`coach chat golden transcripts > 01a tool mode, two rounds, present data "cached": 0, "call": "reconcileSpend", "dateKey": "2026-09-26", - "reserved": 1836, + "reserved": 2286, }, { "call": "appendMessage", @@ -278,7 +278,7 @@ exports[`coach chat golden transcripts > 01b tool mode, present data, one figure ], }, { - "amount": 12891, + "amount": 13341, "call": "reserveBudget", "cap": 2000000, "dateKey": "2026-09-26", @@ -327,7 +327,7 @@ exports[`coach chat golden transcripts > 01b tool mode, present data, one figure "cached": 0, "call": "reconcileSpend", "dateKey": "2026-09-26", - "reserved": 1836, + "reserved": 2286, }, { "call": "annotate", @@ -539,7 +539,7 @@ exports[`coach chat golden transcripts > 02 tool mode, every tool missed, fabric ], }, { - "amount": 12889, + "amount": 13339, "call": "reserveBudget", "cap": 2000000, "dateKey": "2026-09-26", @@ -588,7 +588,7 @@ exports[`coach chat golden transcripts > 02 tool mode, every tool missed, fabric "cached": 0, "call": "reconcileSpend", "dateKey": "2026-09-26", - "reserved": 1836, + "reserved": 2286, }, { "call": "annotate", @@ -1415,7 +1415,7 @@ exports[`coach chat golden transcripts > 07a all providers failed (plain) 1`] = ], }, { - "amount": 12891, + "amount": 13341, "call": "reserveBudget", "cap": 2000000, "dateKey": "2026-09-26", @@ -1453,7 +1453,7 @@ exports[`coach chat golden transcripts > 07a all providers failed (plain) 1`] = "cached": 0, "call": "reconcileSpend", "dateKey": "2026-09-26", - "reserved": 12891, + "reserved": 13341, }, { "call": "annotate", @@ -1507,7 +1507,7 @@ exports[`coach chat golden transcripts > 07b all providers failed (every hop rat ], }, { - "amount": 12891, + "amount": 13341, "call": "reserveBudget", "cap": 2000000, "dateKey": "2026-09-26", @@ -1545,7 +1545,7 @@ exports[`coach chat golden transcripts > 07b all providers failed (every hop rat "cached": 0, "call": "reconcileSpend", "dateKey": "2026-09-26", - "reserved": 12891, + "reserved": 13341, }, { "call": "annotate", @@ -1599,7 +1599,7 @@ exports[`coach chat golden transcripts > 07c all providers failed (primary crede ], }, { - "amount": 12891, + "amount": 13341, "call": "reserveBudget", "cap": 2000000, "dateKey": "2026-09-26", @@ -1637,7 +1637,7 @@ exports[`coach chat golden transcripts > 07c all providers failed (primary crede "cached": 0, "call": "reconcileSpend", "dateKey": "2026-09-26", - "reserved": 12891, + "reserved": 13341, }, { "call": "annotate", @@ -1691,7 +1691,7 @@ exports[`coach chat golden transcripts > 08 an unwrapped tagged provider error 1 ], }, { - "amount": 12891, + "amount": 13341, "call": "reserveBudget", "cap": 2000000, "dateKey": "2026-09-26", @@ -1729,7 +1729,7 @@ exports[`coach chat golden transcripts > 08 an unwrapped tagged provider error 1 "cached": 0, "call": "reconcileSpend", "dateKey": "2026-09-26", - "reserved": 12891, + "reserved": 13341, }, { "call": "annotate", @@ -1784,7 +1784,7 @@ exports[`coach chat golden transcripts > 09 an aborted request leaves a cancelle ], }, { - "amount": 12891, + "amount": 13341, "call": "reserveBudget", "cap": 2000000, "dateKey": "2026-09-26", @@ -1822,7 +1822,7 @@ exports[`coach chat golden transcripts > 09 an aborted request leaves a cancelle "cached": 0, "call": "reconcileSpend", "dateKey": "2026-09-26", - "reserved": 12891, + "reserved": 13341, }, { "call": "appendMessage", @@ -1883,7 +1883,7 @@ exports[`coach chat golden transcripts > 10 outbound block replaces the reply an ], }, { - "amount": 12892, + "amount": 13342, "call": "reserveBudget", "cap": 2000000, "dateKey": "2026-09-26", @@ -1932,7 +1932,7 @@ exports[`coach chat golden transcripts > 10 outbound block replaces the reply an "cached": 0, "call": "reconcileSpend", "dateKey": "2026-09-26", - "reserved": 1836, + "reserved": 2286, }, { "call": "annotate", @@ -2262,7 +2262,7 @@ exports[`coach chat golden transcripts > 11a KEYVALUES valid 1`] = ` ], }, { - "amount": 12891, + "amount": 13341, "call": "reserveBudget", "cap": 2000000, "dateKey": "2026-09-26", @@ -2311,7 +2311,7 @@ exports[`coach chat golden transcripts > 11a KEYVALUES valid 1`] = ` "cached": 0, "call": "reconcileSpend", "dateKey": "2026-09-26", - "reserved": 1836, + "reserved": 2286, }, { "call": "appendMessage", @@ -2515,7 +2515,7 @@ exports[`coach chat golden transcripts > 11b KEYVALUES partial 1`] = ` ], }, { - "amount": 12891, + "amount": 13341, "call": "reserveBudget", "cap": 2000000, "dateKey": "2026-09-26", @@ -2564,7 +2564,7 @@ exports[`coach chat golden transcripts > 11b KEYVALUES partial 1`] = ` "cached": 0, "call": "reconcileSpend", "dateKey": "2026-09-26", - "reserved": 1836, + "reserved": 2286, }, { "call": "annotate", @@ -2766,7 +2766,7 @@ exports[`coach chat golden transcripts > 11c KEYVALUES malformed (no closing mar ], }, { - "amount": 12891, + "amount": 13341, "call": "reserveBudget", "cap": 2000000, "dateKey": "2026-09-26", @@ -2815,7 +2815,7 @@ exports[`coach chat golden transcripts > 11c KEYVALUES malformed (no closing mar "cached": 0, "call": "reconcileSpend", "dateKey": "2026-09-26", - "reserved": 1836, + "reserved": 2286, }, { "call": "annotate", @@ -3001,7 +3001,7 @@ exports[`coach chat golden transcripts > 11d sentinel-only reply is an empty rep ], }, { - "amount": 12891, + "amount": 13341, "call": "reserveBudget", "cap": 2000000, "dateKey": "2026-09-26", @@ -3050,7 +3050,7 @@ exports[`coach chat golden transcripts > 11d sentinel-only reply is an empty rep "cached": 0, "call": "reconcileSpend", "dateKey": "2026-09-26", - "reserved": 1836, + "reserved": 2286, }, { "call": "annotate", @@ -3104,7 +3104,7 @@ exports[`coach chat golden transcripts > 11e empty provider reply 1`] = ` ], }, { - "amount": 12891, + "amount": 13341, "call": "reserveBudget", "cap": 2000000, "dateKey": "2026-09-26", @@ -3153,7 +3153,7 @@ exports[`coach chat golden transcripts > 11e empty provider reply 1`] = ` "cached": 0, "call": "reconcileSpend", "dateKey": "2026-09-26", - "reserved": 1836, + "reserved": 2286, }, ], "contentType": "text/event-stream; charset=utf-8", @@ -3198,7 +3198,7 @@ exports[`coach chat golden transcripts > 12a SUGGEST-REMINDER surfaced 1`] = ` ], }, { - "amount": 12896, + "amount": 13346, "call": "reserveBudget", "cap": 2000000, "dateKey": "2026-09-26", @@ -3247,7 +3247,7 @@ exports[`coach chat golden transcripts > 12a SUGGEST-REMINDER surfaced 1`] = ` "cached": 0, "call": "reconcileSpend", "dateKey": "2026-09-26", - "reserved": 1837, + "reserved": 2287, }, { "call": "prisma.user.update", @@ -3471,7 +3471,7 @@ exports[`coach chat golden transcripts > 12b SUGGEST-REMINDER suppressed 1`] = ` ], }, { - "amount": 12896, + "amount": 13346, "call": "reserveBudget", "cap": 2000000, "dateKey": "2026-09-26", @@ -3520,7 +3520,7 @@ exports[`coach chat golden transcripts > 12b SUGGEST-REMINDER suppressed 1`] = ` "cached": 0, "call": "reconcileSpend", "dateKey": "2026-09-26", - "reserved": 1837, + "reserved": 2287, }, { "call": "annotate", @@ -3708,7 +3708,7 @@ exports[`coach chat golden transcripts > 13 SUGGEST-ACTION 1`] = ` ], }, { - "amount": 12892, + "amount": 13342, "call": "reserveBudget", "cap": 2000000, "dateKey": "2026-09-26", @@ -3757,7 +3757,7 @@ exports[`coach chat golden transcripts > 13 SUGGEST-ACTION 1`] = ` "cached": 0, "call": "reconcileSpend", "dateKey": "2026-09-26", - "reserved": 1836, + "reserved": 2286, }, { "call": "annotate", @@ -3985,7 +3985,7 @@ exports[`coach chat golden transcripts > 14a REMEMBER valid 1`] = ` ], }, { - "amount": 12896, + "amount": 13346, "call": "reserveBudget", "cap": 2000000, "dateKey": "2026-09-26", @@ -4034,7 +4034,7 @@ exports[`coach chat golden transcripts > 14a REMEMBER valid 1`] = ` "cached": 0, "call": "reconcileSpend", "dateKey": "2026-09-26", - "reserved": 1837, + "reserved": 2287, }, { "call": "captureReminderFromSentinel", @@ -4242,7 +4242,7 @@ exports[`coach chat golden transcripts > 14b REMEMBER malformed 1`] = ` ], }, { - "amount": 12896, + "amount": 13346, "call": "reserveBudget", "cap": 2000000, "dateKey": "2026-09-26", @@ -4291,7 +4291,7 @@ exports[`coach chat golden transcripts > 14b REMEMBER malformed 1`] = ` "cached": 0, "call": "reconcileSpend", "dateKey": "2026-09-26", - "reserved": 1837, + "reserved": 2287, }, { "call": "annotate", @@ -4494,7 +4494,7 @@ exports[`coach chat golden transcripts > 15 an unknown learn link is scrubbed, a ], }, { - "amount": 12895, + "amount": 13345, "call": "reserveBudget", "cap": 2000000, "dateKey": "2026-09-26", @@ -4543,7 +4543,7 @@ exports[`coach chat golden transcripts > 15 an unknown learn link is scrubbed, a "cached": 0, "call": "reconcileSpend", "dateKey": "2026-09-26", - "reserved": 1837, + "reserved": 2287, }, { "call": "annotate", @@ -5207,7 +5207,7 @@ exports[`coach chat golden transcripts > 17a a workout-scoped first turn 1`] = ` ], }, { - "amount": 12890, + "amount": 13340, "call": "reserveBudget", "cap": 2000000, "dateKey": "2026-09-26", @@ -5256,7 +5256,7 @@ exports[`coach chat golden transcripts > 17a a workout-scoped first turn 1`] = ` "cached": 0, "call": "reconcileSpend", "dateKey": "2026-09-26", - "reserved": 1836, + "reserved": 2286, }, { "call": "appendMessage", @@ -5465,7 +5465,7 @@ exports[`coach chat golden transcripts > 17b a workout-scoped first turn with th ], }, { - "amount": 12890, + "amount": 13340, "call": "reserveBudget", "cap": 2000000, "dateKey": "2026-09-26", @@ -5514,7 +5514,7 @@ exports[`coach chat golden transcripts > 17b a workout-scoped first turn with th "cached": 0, "call": "reconcileSpend", "dateKey": "2026-09-26", - "reserved": 1836, + "reserved": 2286, }, { "call": "appendMessage", @@ -5702,7 +5702,7 @@ exports[`coach chat golden transcripts > 18 a persistence throw after the provid ], }, { - "amount": 12891, + "amount": 13341, "call": "reserveBudget", "cap": 2000000, "dateKey": "2026-09-26", @@ -5751,7 +5751,7 @@ exports[`coach chat golden transcripts > 18 a persistence throw after the provid "cached": 0, "call": "reconcileSpend", "dateKey": "2026-09-26", - "reserved": 1836, + "reserved": 2286, }, { "call": "event.setError", @@ -5817,7 +5817,7 @@ exports[`coach chat golden transcripts > 19 an existing conversation with prior ], }, { - "amount": 13053, + "amount": 13503, "call": "reserveBudget", "cap": 2000000, "dateKey": "2026-09-26", @@ -5866,7 +5866,7 @@ exports[`coach chat golden transcripts > 19 an existing conversation with prior "cached": 0, "call": "reconcileSpend", "dateKey": "2026-09-26", - "reserved": 1851, + "reserved": 2301, }, { "call": "annotate", @@ -6078,7 +6078,7 @@ exports[`coach chat golden transcripts > 20 a guided-question turn with a scope ], }, { - "amount": 12892, + "amount": 13342, "call": "reserveBudget", "cap": 2000000, "dateKey": "2026-09-26", @@ -6127,7 +6127,7 @@ exports[`coach chat golden transcripts > 20 a guided-question turn with a scope "cached": 0, "call": "reconcileSpend", "dateKey": "2026-09-26", - "reserved": 1836, + "reserved": 2286, }, { "call": "appendMessage", @@ -6295,7 +6295,7 @@ exports[`coach chat golden transcripts > 21 the German locale with a stored self ], }, { - "amount": 12917, + "amount": 13367, "call": "reserveBudget", "cap": 2000000, "dateKey": "2026-09-26", @@ -6344,7 +6344,7 @@ exports[`coach chat golden transcripts > 21 the German locale with a stored self "cached": 0, "call": "reconcileSpend", "dateKey": "2026-09-26", - "reserved": 1839, + "reserved": 2289, }, { "call": "appendMessage", @@ -6548,7 +6548,7 @@ exports[`coach chat golden transcripts > 22 tool mode with live steps: found, em ], }, { - "amount": 12891, + "amount": 13341, "call": "reserveBudget", "cap": 2000000, "dateKey": "2026-09-26", @@ -6597,7 +6597,7 @@ exports[`coach chat golden transcripts > 22 tool mode with live steps: found, em "cached": 0, "call": "reconcileSpend", "dateKey": "2026-09-26", - "reserved": 1836, + "reserved": 2286, }, { "call": "appendMessage", diff --git a/src/app/api/insights/chat/__tests__/dialog-harness.ts b/src/app/api/insights/chat/__tests__/dialog-harness.ts index 14f787db9..c9b2dc079 100644 --- a/src/app/api/insights/chat/__tests__/dialog-harness.ts +++ b/src/app/api/insights/chat/__tests__/dialog-harness.ts @@ -63,6 +63,7 @@ export interface ProviderCall { system: string; messages: Array<{ role: string; content: string }>; toolChoice: string | undefined; + cacheKey: string | undefined; toolCalls: Array<{ name: string; args: Record }>; } export const providerCalls: ProviderCall[] = []; @@ -278,6 +279,7 @@ async function runRawCompletionWithFallback(args: { system: string; messages: Array<{ role: string; content: string }>; toolChoice?: string; + cacheKey?: string; }; }) { const round = world.script.shift() ?? { text: "" }; @@ -289,6 +291,7 @@ async function runRawCompletionWithFallback(args: { content: msg.content, })), toolChoice: args.params.toolChoice, + cacheKey: args.params.cacheKey, toolCalls: calls, }); const offered = args.params.toolChoice !== "none"; diff --git a/src/app/api/insights/chat/__tests__/route-activity.test.ts b/src/app/api/insights/chat/__tests__/route-activity.test.ts index 92cb32fe5..04c689803 100644 --- a/src/app/api/insights/chat/__tests__/route-activity.test.ts +++ b/src/app/api/insights/chat/__tests__/route-activity.test.ts @@ -279,7 +279,9 @@ describe("concurrent turns", () => { expect(m.checkRateLimit).toHaveBeenCalledWith( "coach-turn-active:u1", 2, - 180_000, + // The longest turn's wall time (200 s since v1.41.2) plus 30 s for + // its reply to stream. + 230_000, ); }); }); diff --git a/src/app/api/insights/chat/route.ts b/src/app/api/insights/chat/route.ts index 971f391f8..b55b23d4f 100644 --- a/src/app/api/insights/chat/route.ts +++ b/src/app/api/insights/chat/route.ts @@ -40,6 +40,7 @@ import { runCoachTurn } from "@/lib/ai/coach/turn/pipeline"; import type { ReasoningLevel } from "@/lib/ai/reasoning/levels"; import { resolveCoachTurnReasoningLevel } from "@/lib/ai/reasoning/controls"; import { streamProviderError, streamRefusal } from "@/lib/ai/coach/turn/sse"; +import { TURN_LIMITS } from "@/lib/ai/coach/tools/turn-budget"; /** * The `coach` capability gate for a turn. Every unavailable reason throws the @@ -72,11 +73,12 @@ async function coachCapabilityRefusal(): Promise { */ const COACH_CONCURRENT_TURNS = 2; /** - * How long a turn's slot is held at most: a turn's wall time (150 s) plus - * the streaming of its reply. A slot whose release never arrived (a crashed - * process) frees itself when the window ends. + * How long a turn's slot is held at most: the longest turn's wall time + * (`TURN_LIMITS.user.wallMs`, 200 s) plus 30 s for the streaming of its + * reply. A slot whose release never arrived (a crashed process) frees itself + * when the window ends. */ -const COACH_TURN_SLOT_MS = 180_000; +const COACH_TURN_SLOT_MS = TURN_LIMITS.user.wallMs + 30_000; async function handleChatRequest(request: NextRequest): Promise { const auth = await requireAuth(); diff --git a/src/lib/ai/__tests__/cached-input-usage.test.ts b/src/lib/ai/__tests__/cached-input-usage.test.ts new file mode 100644 index 000000000..2a555aad5 --- /dev/null +++ b/src/lib/ai/__tests__/cached-input-usage.test.ts @@ -0,0 +1,368 @@ +/** + * Cached input, per provider, end to end into the Coach turn budget. + * + * The turn budget counts cached input at a tenth (`CACHED_INPUT_WEIGHT`). + * That only works when two things hold for every client: + * + * 1. `tokensUsed` is the gross count, cached input included, and + * `cachedInputTokens` is the cached share of it. The budget and the + * daily ledger both subtract the cached share from the gross. + * 2. The cached count is read from where the provider puts it: Codex and + * the Responses wire under `usage.input_tokens_details.cached_tokens`, + * Chat Completions (OpenAI, gateways, local servers that report it) + * under `usage.prompt_tokens_details.cached_tokens`, Anthropic as + * `usage.cache_read_input_tokens` beside an `input_tokens` that does + * NOT include it. + * + * Each fixture below is a usage payload in the provider's own shape for the + * same round: 20,000 tokens of input of which 18,000 came from the cache, + * and 500 of output. Every client must turn it into gross 20,500 / cached + * 18,000, which the budget weighs at 2,000 + 500 + 1,800 = 4,300. + * + * And the cache has to be hit at all: the Codex backend routes a request to + * a cache by `prompt_cache_key` (the official client sets it to its thread + * id), and `api.openai.com` documents the same field. A turn's rounds share + * one key, so round two finds round one's prefix. + */ +import { beforeEach, describe, expect, it, vi } from "vitest"; + +vi.mock("undici", async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + fetch: (input: unknown, init?: unknown) => + (globalThis.fetch as unknown as (i: unknown, n?: unknown) => unknown)( + input, + init, + ), + }; +}); +vi.mock("../local-host-allowlist", () => ({ aiEgressPolicyFor: () => ({}) })); + +import { AnthropicClient } from "../anthropic-client"; +import { CodexClient } from "../codex-client"; +import { OpenAIClient } from "../openai-client"; +import { LocalOpenAICompatibleClient } from "../local-client"; +import { singleUserTurn, type CompletionResult } from "../types"; +import { weightedRoundTokens } from "@/lib/ai/coach/tools/turn-budget"; + +const INPUT = 20_000; +const CACHED = 18_000; +const OUTPUT = 500; +const GROSS = INPUT + OUTPUT; +const WEIGHTED = INPUT - CACHED + OUTPUT + CACHED / 10; + +function expectDiscounted(result: CompletionResult) { + expect(result.tokensUsed).toBe(GROSS); + expect(result.cachedInputTokens).toBe(CACHED); + expect( + weightedRoundTokens({ + tokens: result.tokensUsed, + cachedTokens: result.cachedInputTokens, + }), + ).toBe(WEIGHTED); +} + +function sse(events: string[]): Response { + const encoder = new TextEncoder(); + return new Response( + new ReadableStream({ + start(controller) { + for (const ev of events) controller.enqueue(encoder.encode(ev)); + controller.close(); + }, + }), + { status: 200, headers: { "content-type": "text/event-stream" } }, + ); +} + +function codexStream(): Response { + return sse([ + `event: response.output_item.done\ndata: ${JSON.stringify({ + type: "response.output_item.done", + item: { + type: "message", + role: "assistant", + content: [{ type: "output_text", text: "ok" }], + }, + })}\n\n`, + `event: response.completed\ndata: ${JSON.stringify({ + type: "response.completed", + response: { + id: "resp1", + usage: { + input_tokens: INPUT, + input_tokens_details: { cached_tokens: CACHED }, + output_tokens: OUTPUT, + output_tokens_details: { reasoning_tokens: 0 }, + total_tokens: GROSS, + }, + }, + })}\n\n`, + ]); +} + +/** The Chat Completions usage block, as OpenAI and most gateways send it. */ +const chatUsage = { + prompt_tokens: INPUT, + completion_tokens: OUTPUT, + total_tokens: GROSS, + prompt_tokens_details: { cached_tokens: CACHED }, +}; + +function jsonResponse(body: unknown) { + return { + ok: true, + status: 200, + headers: { get: () => "application/json" }, + json: () => Promise.resolve(body), + text: () => Promise.resolve(JSON.stringify(body)), + }; +} + +function codexClient() { + return new CodexClient({ + accessToken: "t", + accountId: "a", + onTokenRefresh: vi + .fn() + .mockResolvedValue({ accessToken: "t", accountId: "a" }), + slugChain: ["gpt-5-codex"], + }); +} + +beforeEach(() => { + vi.restoreAllMocks(); +}); + +describe("cached input reaches the turn budget, per provider", () => { + it("codex: input_tokens_details.cached_tokens", async () => { + vi.stubGlobal("fetch", vi.fn().mockResolvedValue(codexStream())); + const result = await codexClient().generateCompletion( + singleUserTurn({ system: "s", user: "u" }), + ); + expectDiscounted(result); + }); + + it("openai (Chat Completions): prompt_tokens_details.cached_tokens", async () => { + vi.stubGlobal( + "fetch", + vi.fn().mockResolvedValue( + jsonResponse({ + choices: [{ message: { content: "ok" }, finish_reason: "stop" }], + usage: chatUsage, + }), + ), + ); + const client = new OpenAIClient({ + apiKey: "sk-test", + model: "gpt-4o-mini", + baseUrl: "https://api.openai.com/v1", + }); + expectDiscounted( + await client.generateCompletion( + singleUserTurn({ system: "s", user: "u" }), + ), + ); + }); + + it("gateway (OpenRouter and the like): the same Chat Completions block", async () => { + vi.stubGlobal( + "fetch", + vi.fn().mockResolvedValue( + jsonResponse({ + choices: [{ message: { content: "ok" }, finish_reason: "stop" }], + usage: chatUsage, + }), + ), + ); + const client = new OpenAIClient({ + apiKey: "sk-or-test", + model: "openai/gpt-4o-mini", + baseUrl: "https://openrouter.ai/api/v1", + providerType: "openai-compatible", + }); + expectDiscounted( + await client.generateCompletion( + singleUserTurn({ system: "s", user: "u" }), + ), + ); + }); + + it("anthropic: cache reads ride beside input_tokens, not inside it", async () => { + vi.stubGlobal( + "fetch", + vi.fn().mockResolvedValue( + jsonResponse({ + content: [{ type: "text", text: "ok" }], + stop_reason: "end_turn", + usage: { + // Anthropic's input_tokens counts only what came after the last + // cache breakpoint; reads and writes are reported beside it. + input_tokens: INPUT - CACHED, + cache_read_input_tokens: CACHED, + cache_creation_input_tokens: 0, + output_tokens: OUTPUT, + }, + }), + ), + ); + const client = new AnthropicClient({ + apiKey: "sk-ant-test", + model: "claude-sonnet-4-5", + }); + expectDiscounted( + await client.generateCompletion( + singleUserTurn({ system: "s", user: "u" }), + ), + ); + }); + + it("anthropic: a cache write counts in full", async () => { + vi.stubGlobal( + "fetch", + vi.fn().mockResolvedValue( + jsonResponse({ + content: [{ type: "text", text: "ok" }], + stop_reason: "end_turn", + usage: { + input_tokens: 2_000, + cache_read_input_tokens: 0, + cache_creation_input_tokens: 18_000, + output_tokens: OUTPUT, + }, + }), + ), + ); + const client = new AnthropicClient({ + apiKey: "sk-ant-test", + model: "claude-sonnet-4-5", + }); + const result = await client.generateCompletion( + singleUserTurn({ system: "s", user: "u" }), + ); + expect(result.tokensUsed).toBe(GROSS); + expect(result.cachedInputTokens ?? 0).toBe(0); + }); + + it("local: prompt_tokens_details.cached_tokens where the server reports it", async () => { + vi.stubGlobal( + "fetch", + vi.fn().mockResolvedValue( + jsonResponse({ + choices: [{ message: { content: "ok" }, finish_reason: "stop" }], + usage: chatUsage, + }), + ), + ); + const client = new LocalOpenAICompatibleClient({ + apiKey: null, + model: "llama3:8b", + baseUrl: "http://localhost:11434/v1", + }); + expectDiscounted( + await client.generateCompletion( + singleUserTurn({ system: "s", user: "u" }), + ), + ); + }); + + it("local, streamed: the usage frame's cached count", async () => { + vi.stubGlobal( + "fetch", + vi + .fn() + .mockResolvedValue( + sse([ + 'data: {"choices":[{"delta":{"content":"ok"}}]}\n\n', + `data: ${JSON.stringify({ choices: [{ delta: {}, finish_reason: "stop" }], usage: chatUsage })}\n\n`, + "data: [DONE]\n\n", + ]), + ), + ); + const client = new LocalOpenAICompatibleClient({ + apiKey: null, + model: "llama3:8b", + baseUrl: "http://localhost:11434/v1", + }); + expectDiscounted( + await client.generateCompletionStream( + singleUserTurn({ system: "s", user: "u" }), + () => {}, + ), + ); + }); +}); + +describe("a turn's rounds share one prompt cache key", () => { + it("codex sends prompt_cache_key and one session id for every round with the same key", async () => { + const fetchMock = vi + .fn() + .mockImplementation(() => Promise.resolve(codexStream())); + vi.stubGlobal("fetch", fetchMock); + const client = codexClient(); + const params = { + ...singleUserTurn({ system: "s", user: "u" }), + cacheKey: "coach-turn-key", + }; + await client.generateCompletion(params); + await client.generateCompletion(params); + + const [first, second] = fetchMock.mock.calls.map( + (call) => call[1] as { body: string; headers: Record }, + ); + expect(JSON.parse(first.body).prompt_cache_key).toBe("coach-turn-key"); + expect(JSON.parse(second.body).prompt_cache_key).toBe("coach-turn-key"); + expect(first.headers.session_id).toBe(second.headers.session_id); + expect(first.headers.session_id).toMatch( + /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/, + ); + }); + + it("codex without a key sends none and keeps a fresh session per call", async () => { + const fetchMock = vi + .fn() + .mockImplementation(() => Promise.resolve(codexStream())); + vi.stubGlobal("fetch", fetchMock); + const client = codexClient(); + await client.generateCompletion(singleUserTurn({ system: "s", user: "u" })); + await client.generateCompletion(singleUserTurn({ system: "s", user: "u" })); + const [first, second] = fetchMock.mock.calls.map( + (call) => call[1] as { body: string; headers: Record }, + ); + expect("prompt_cache_key" in JSON.parse(first.body)).toBe(false); + expect(first.headers.session_id).not.toBe(second.headers.session_id); + }); + + it("api.openai.com gets prompt_cache_key; a gateway never does", async () => { + const reply = () => + Promise.resolve( + jsonResponse({ + choices: [{ message: { content: "ok" }, finish_reason: "stop" }], + usage: chatUsage, + }), + ); + const fetchMock = vi.fn().mockImplementation(reply); + vi.stubGlobal("fetch", fetchMock); + const params = { + ...singleUserTurn({ system: "s", user: "u" }), + cacheKey: "coach-turn-key", + }; + await new OpenAIClient({ + apiKey: "sk-test", + model: "gpt-4o-mini", + baseUrl: "https://api.openai.com/v1", + }).generateCompletion(params); + await new OpenAIClient({ + apiKey: "sk-or-test", + model: "openai/gpt-4o-mini", + baseUrl: "https://openrouter.ai/api/v1", + providerType: "openai-compatible", + }).generateCompletion(params); + const [direct, gateway] = fetchMock.mock.calls.map((call) => + JSON.parse((call[1] as { body: string }).body), + ); + expect(direct.prompt_cache_key).toBe("coach-turn-key"); + expect("prompt_cache_key" in gateway).toBe(false); + }); +}); diff --git a/src/lib/ai/anthropic-client.ts b/src/lib/ai/anthropic-client.ts index 654de706b..79a681234 100644 --- a/src/lib/ai/anthropic-client.ts +++ b/src/lib/ai/anthropic-client.ts @@ -412,6 +412,7 @@ export class AnthropicClient implements AIProvider { input_tokens?: number; output_tokens?: number; cache_read_input_tokens?: number; + cache_creation_input_tokens?: number; output_tokens_details?: { thinking_tokens?: number }; }; }; @@ -487,9 +488,19 @@ export class AnthropicClient implements AIProvider { const content = wantsJson && rawText ? extractJsonObject(rawText) : (rawText ?? ""); + // v1.41.2 — the gross count includes the cached prefix. Anthropic reports + // cache reads and cache writes BESIDE `input_tokens`, which holds only + // what came after the last breakpoint; every other client reports a total + // that already contains the cached share. The turn budget and the daily + // ledger subtract `cachedInputTokens` from `tokensUsed`, so leaving the + // reads out of the gross made a mostly cached round look nearly free and + // a cache write (billed above the base rate) went uncounted. const inputTokens = json.usage?.input_tokens ?? 0; + const cacheReadTokens = json.usage?.cache_read_input_tokens ?? 0; + const cacheWriteTokens = json.usage?.cache_creation_input_tokens ?? 0; const outputTokens = json.usage?.output_tokens ?? 0; - const tokensUsed = inputTokens + outputTokens || null; + const tokensUsed = + inputTokens + cacheReadTokens + cacheWriteTokens + outputTokens || null; const cachedInputTokens = json.usage?.cache_read_input_tokens ?? null; // The presence of tool_use blocks is authoritative for the tool-loop gate: diff --git a/src/lib/ai/coach/eval/__tests__/scenarios-live.test.ts b/src/lib/ai/coach/eval/__tests__/scenarios-live.test.ts index ecf3383e1..61891818d 100644 --- a/src/lib/ai/coach/eval/__tests__/scenarios-live.test.ts +++ b/src/lib/ai/coach/eval/__tests__/scenarios-live.test.ts @@ -5,6 +5,7 @@ * chat turn's own context, the reply through the turn's parsers) and the * grade, so a live run measures the model and nothing else. */ +import { TURN_LIMITS } from "@/lib/ai/coach/tools/turn-budget"; import { afterEach, describe, expect, it, vi } from "vitest"; import type { CompletionParams, CompletionResult } from "@/lib/ai/types"; @@ -119,23 +120,27 @@ describe("runScenarioLive", () => { it("forces an answer at the round cap of the person's own plan", async () => { const call = toolRound("get_metric_table", { metric: "pulse" }); + // v1.41.2 — the cap is sixteen (was twelve); read from the limits so + // the scenario run and the turn cannot drift apart again. + const cap = TURN_LIMITS.user.maxRounds; + expect(cap).toBe(16); const provider = scripted([ - ...Array.from({ length: 11 }, () => call), + ...Array.from({ length: cap - 1 }, () => call), { content: "So far." }, ]); const observation = await runScenarioLive({ scenario: scenario("forced-final.en"), provider, }); - expect(observation.providerCalls).toBe(12); + expect(observation.providerCalls).toBe(cap); expect(provider.sent.map((p) => p.toolChoice)).toEqual([ - ...Array.from({ length: 11 }, () => "auto"), + ...Array.from({ length: cap - 1 }, () => "auto"), "none", ]); expect(observation.prose).toBe("So far."); - expect(observation.toolRounds).toBe(11); + expect(observation.toolRounds).toBe(cap - 1); // The same call every round: graded as repeats. - expect(observation.repeatedCalls).toBe(10); + expect(observation.repeatedCalls).toBe(cap - 2); }); it("ends the run on a clarifying question asked through the tool", async () => { diff --git a/src/lib/ai/coach/eval/__tests__/scenarios.test.ts b/src/lib/ai/coach/eval/__tests__/scenarios.test.ts index 9b077b2fc..bfdbc0325 100644 --- a/src/lib/ai/coach/eval/__tests__/scenarios.test.ts +++ b/src/lib/ai/coach/eval/__tests__/scenarios.test.ts @@ -63,6 +63,7 @@ import type { CoachStep, } from "@/lib/ai/coach/types"; import { COACH_FOLLOW_UP_KEYS } from "@/lib/ai/coach/dialog-keys"; +import { TURN_LIMITS } from "@/lib/ai/coach/tools/turn-budget"; import { POST } from "@/app/api/insights/chat/route"; import { @@ -130,7 +131,7 @@ const call = (name: string, args: Record = {}) => ({ }); /** Ten distinct reads: with `get_sleep` first, eleven fetching rounds. */ -const FORCED_READS: ReadonlyArray<[string, string]> = [ +const FORCED_READS: ReadonlyArray<[string, string, string?]> = [ ["pulse", "last7days"], ["pulse", "last30days"], ["pulse", "last90days"], @@ -141,6 +142,11 @@ const FORCED_READS: ReadonlyArray<[string, string]> = [ ["bp", "lastYear"], ["sleep", "last90days"], ["sleep", "lastYear"], + // v1.41.2 — four more for the sixteen-round cap. + ["sleep", "last7days"], + ["pulse", "last30days", "previous"], + ["sleep", "last30days"], + ["bp", "last30days", "previous"], ]; function script(name: string, lang: Lang): Round[] { @@ -204,12 +210,18 @@ function script(name: string, lang: Lang): Round[] { return []; case "forced-final": // v1.41 — a new read every round, never a repeat, until the round cap - // of the person's own plan (twelve, the forced answer included) makes - // the twelfth round the answer. + // of the person's own plan (sixteen since v1.41.2, the forced answer + // included) makes the last round the answer. return [ { calls: [call("get_sleep")] }, - ...FORCED_READS.map(([metric, window]) => ({ - calls: [call("get_metric_table", { metric, window })], + ...FORCED_READS.map(([metric, window, period]) => ({ + calls: [ + call("get_metric_table", { + metric, + window, + ...(period ? { period } : {}), + }), + ], })), { text: PROSE.forced[lang] }, ]; @@ -766,9 +778,14 @@ describe.each(["de", "en"] as const)("in %s", (lang) => { const scenario = byName("forced-final", lang); const { observation, frames } = await run(scenario); expect(providerCalls.map((c) => c.toolChoice)).toEqual([ - ...Array.from({ length: 11 }, () => "auto"), + ...Array.from({ length: TURN_LIMITS.user.maxRounds - 1 }, () => "auto"), "none", ]); + // v1.41.2 — every round of the turn carries the same prompt-cache key, + // so a provider that routes by key finds the prefix the last round left. + const keys = new Set(providerCalls.map((c) => c.cacheKey)); + expect(keys.size).toBe(1); + expect([...keys][0]).toMatch(/^[0-9a-f]{32}$/); const provenance = framesOf<{ metricSource: { forcedFinal?: boolean } }>( frames, "provenance", diff --git a/src/lib/ai/coach/eval/run-case.ts b/src/lib/ai/coach/eval/run-case.ts index f6d2856b0..9eecf3986 100644 --- a/src/lib/ai/coach/eval/run-case.ts +++ b/src/lib/ai/coach/eval/run-case.ts @@ -26,6 +26,7 @@ */ import type { Locale } from "@/lib/i18n/config"; import { UNBOUNDED_REACH } from "@/lib/ai/coach/history-reach"; +import { TURN_LIMITS } from "@/lib/ai/coach/tools/turn-budget"; import type { AiMessage, AiToolCall, @@ -145,7 +146,7 @@ export interface ScenarioProvider { * How many rounds a live scenario may take: the round cap of the person's * own plan (v1.41), the forced answer included. */ -const SCENARIO_MAX_ROUNDS = 12; +const SCENARIO_MAX_ROUNDS = TURN_LIMITS.user.maxRounds; function parseArgs(raw: string): Record { try { @@ -236,7 +237,7 @@ function priorResultTurns(scenario: CoachScenario) { * the chat turn's own context (system prompt, tool-mode and dialog * addenda, DATA INVENTORY, EARLIER TABLES, transcript), put to a real * model over the real tool catalogue, comparisons and the dialog tools - * for up to twelve rounds (v1.41), a question ending the run. Every call is + * for as many rounds as a turn on the person's own plan (v1.41), a question ending the run. Every call is * captured with its arguments and answered with a stand-in result; the * reply goes through the clarification and chip parsers the turn uses. * diff --git a/src/lib/ai/coach/prompt-cache-key.ts b/src/lib/ai/coach/prompt-cache-key.ts new file mode 100644 index 000000000..c70a12e96 --- /dev/null +++ b/src/lib/ai/coach/prompt-cache-key.ts @@ -0,0 +1,20 @@ +/** + * v1.41.2 — the prompt-cache key of a Coach conversation. + * + * Every round of a turn resends the same system prompt and history, and the + * next turn of the conversation resends it again with one more exchange. A + * provider that routes requests to its prompt cache by key (Codex, + * `api.openai.com`) only finds the prefix an earlier call left there when the + * calls carry the same key. One key per conversation does that; a turn with + * no conversation yet gets a key of its own, still shared by its rounds. + * + * Hashed so the provider never sees the conversation id itself. + */ +import { createHash, randomUUID } from "node:crypto"; + +export function coachPromptCacheKey(conversationId?: string | null): string { + return createHash("sha256") + .update(`coach-prompt:${conversationId || randomUUID()}`) + .digest("hex") + .slice(0, 32); +} diff --git a/src/lib/ai/coach/stream-events.ts b/src/lib/ai/coach/stream-events.ts index 5c54094bb..c0757bc43 100644 --- a/src/lib/ai/coach/stream-events.ts +++ b/src/lib/ai/coach/stream-events.ts @@ -76,7 +76,7 @@ export const coachStepSchema = z id: z .string() .regex(/^s[1-9]\d{0,2}$/) - .describe("`s1`..`s48`, unique within a turn."), + .describe("`s1`..`s64`, unique within a turn."), tool: z.enum([...COACH_TOOL_NAMES, "show_result", "snapshot"]), labelKey: z .string() diff --git a/src/lib/ai/coach/tools/__tests__/turn-budget.test.ts b/src/lib/ai/coach/tools/__tests__/turn-budget.test.ts index 2a251d442..b43779869 100644 --- a/src/lib/ai/coach/tools/__tests__/turn-budget.test.ts +++ b/src/lib/ai/coach/tools/__tests__/turn-budget.test.ts @@ -25,18 +25,40 @@ function clock(start = 0) { } describe("TURN_LIMITS", () => { + // v1.41.2 — raised by about a third from 120k / 150 s / 12 and + // 40k / 90 s / 6: a turn on a person's own plan stopped for tokens after + // five rounds while it was still reading. it("gives the person's own plan more than the operator's", () => { expect(TURN_LIMITS.user).toEqual({ - tokens: 120_000, - wallMs: 150_000, - maxRounds: 12, + tokens: 165_000, + wallMs: 200_000, + maxRounds: 16, }); expect(TURN_LIMITS.operator).toEqual({ - tokens: 40_000, - wallMs: 90_000, - maxRounds: 6, + tokens: 55_000, + wallMs: 120_000, + maxRounds: 8, }); }); + + it("keeps the final answer at the same share of each payer's tokens", () => { + expect(FINAL_ANSWER_TOKENS / TURN_LIMITS.user.tokens).toBeCloseTo(0.01); + expect(FINAL_ANSWER_TOKENS / TURN_LIMITS.operator.tokens).toBeCloseTo(0.03); + }); + + it("holds the turn's slot for at least the longest turn", async () => { + // The chat route's concurrency slot frees itself after its window; a + // window shorter than the wall time would let a third turn start while + // a long one is still running. + const { readFileSync } = await import("node:fs"); + const route = readFileSync( + `${process.cwd()}/src/app/api/insights/chat/route.ts`, + "utf8", + ); + expect(route).toMatch( + /COACH_TURN_SLOT_MS = TURN_LIMITS\.user\.wallMs \+ \d/, + ); + }); }); describe("token accounting", () => { @@ -73,9 +95,9 @@ describe("check()", () => { effort: "medium", initialInputTokens: 10_000, }); - // 34k spent; the next round with medium thinking and the final answer - // need about 8k more, past the operator's 40k. - budget.endRound({ tokens: 34_000, cachedTokens: 0, durationMs: 1_000 }); + // 49k spent; the next round with medium thinking and the final answer + // need about 8k more, past the operator's 55k. + budget.endRound({ tokens: 49_000, cachedTokens: 0, durationMs: 1_000 }); expect(budget.check()).toBe("budget"); }); @@ -103,9 +125,9 @@ describe("check()", () => { initialInputTokens: 1_000, now: c.now, }); - c.advance(30_000); + c.advance(60_000); budget.endRound({ tokens: 1_000, cachedTokens: 0, durationMs: 30_000 }); - // 30 s elapsed + two typical 30 s rounds = 90 s: not over yet. + // 60 s elapsed + two typical 30 s rounds = 120 s: not over yet. expect(budget.check()).toBeNull(); c.advance(1); budget.endRound({ tokens: 1_000, cachedTokens: 900, durationMs: 30_000 }); diff --git a/src/lib/ai/coach/tools/loop.ts b/src/lib/ai/coach/tools/loop.ts index f784a9096..ea45a44ad 100644 --- a/src/lib/ai/coach/tools/loop.ts +++ b/src/lib/ai/coach/tools/loop.ts @@ -30,6 +30,7 @@ * A clarifying question ends the turn without a prose round: the question * is the reply. */ +import { coachPromptCacheKey } from "@/lib/ai/coach/prompt-cache-key"; import type { CoachHistoryReach } from "@/lib/ai/coach/history-reach"; import { annotate } from "@/lib/logging/context"; import type { Locale } from "@/lib/i18n/config"; @@ -277,6 +278,9 @@ export async function runCoachToolLoop(args: { ), }); const progress = createProgressTracker(); + // v1.41.2 — one key for every round, so round two finds round one's prefix + // in the provider's prompt cache. + const cacheKey = coachPromptCacheKey(turn?.conversationId); const messages: AiMessage[] = [...args.messages]; let system = args.system; @@ -351,6 +355,7 @@ export async function runCoachToolLoop(args: { : {}), tools, toolChoice: isFinal ? ("none" as const) : ("auto" as const), + cacheKey, }, }); const result = fallback.result; diff --git a/src/lib/ai/coach/tools/turn-budget.ts b/src/lib/ai/coach/tools/turn-budget.ts index 3d021441c..16f8ba986 100644 --- a/src/lib/ai/coach/tools/turn-budget.ts +++ b/src/lib/ai/coach/tools/turn-budget.ts @@ -46,17 +46,26 @@ export interface TurnLimits { * Fixed per payer, not configurable: the person's own plan, key or local * model gets room for a long "why" chain; a turn the operator pays for gets * a third of the tokens and half the rounds. + * + * v1.41.2 — about a third more on both sides (was 120k / 150 s / 12 rounds + * and 40k / 90 s / 6 rounds). A real turn on a person's own plan stopped for + * tokens after five rounds while it was still reading, and the operator + * side moves with it so the split stays the same. */ export const TURN_LIMITS: Readonly> = { - user: { tokens: 120_000, wallMs: 150_000, maxRounds: 12 }, - operator: { tokens: 40_000, wallMs: 90_000, maxRounds: 6 }, + user: { tokens: 165_000, wallMs: 200_000, maxRounds: 16 }, + operator: { tokens: 55_000, wallMs: 120_000, maxRounds: 8 }, }; /** The answer budget of one round, before any thinking budget. */ export const ROUND_ANSWER_TOKENS = 600; -/** What the final answer is given on top of its input. */ -export const FINAL_ANSWER_TOKENS = 1_200; +/** + * What the final answer is given on top of its input. v1.41.2 — raised with + * the limits (was 1,200) so it keeps the same share of each payer's tokens: + * 1 % of the person's own budget, 3 % of the operator's. + */ +export const FINAL_ANSWER_TOKENS = 1_650; /** Cached input counts at this share of its size. */ export const CACHED_INPUT_WEIGHT = 0.1; diff --git a/src/lib/ai/coach/turn/__tests__/steps.test.ts b/src/lib/ai/coach/turn/__tests__/steps.test.ts index af70675a1..ccd097b50 100644 --- a/src/lib/ai/coach/turn/__tests__/steps.test.ts +++ b/src/lib/ai/coach/turn/__tests__/steps.test.ts @@ -462,7 +462,7 @@ describe("toStep — mapping", () => { parsedArgs: parsed, locale: "en", })?.id, - ).toBe("s48"); + ).toBe("s64"); expect( toStep({ call: { id: "u", name: "get_everything", arguments: "{}" }, diff --git a/src/lib/ai/coach/turn/model.ts b/src/lib/ai/coach/turn/model.ts index dc2a44810..f8c7996f7 100644 --- a/src/lib/ai/coach/turn/model.ts +++ b/src/lib/ai/coach/turn/model.ts @@ -4,6 +4,7 @@ * completion over the full snapshot. A provider failure is classified into * the structured `coach.*` code the stream answers with. */ +import { coachPromptCacheKey } from "@/lib/ai/coach/prompt-cache-key"; import { annotate } from "@/lib/logging/context"; import type { Locale } from "@/lib/i18n/config"; import { localeLanguageNames as LANGUAGE_NAMES } from "@/lib/i18n/config"; @@ -652,6 +653,7 @@ export async function runTurnModel(args: { // v1.22 (#89) — per-idle-gap timeout for the streaming local call. timeoutMs: ctx.aiResponseTimeoutMs, }), + cacheKey: coachPromptCacheKey(conversationId), ...(args.reasoning ? { reasoning: args.reasoning, diff --git a/src/lib/ai/coach/turn/steps.ts b/src/lib/ai/coach/turn/steps.ts index 2543185db..d09ce5d3f 100644 --- a/src/lib/ai/coach/turn/steps.ts +++ b/src/lib/ai/coach/turn/steps.ts @@ -40,9 +40,10 @@ import { COMPARE_SERIES_TOOL_NAME } from "@/lib/ai/coach/tools/compare-series"; /** * At most this many steps per turn; later calls run but show no step. - * v1.41 — 48: a budgeted turn runs up to twelve rounds. + * v1.41 — four per round of the longest turn: 48 for twelve rounds, 64 since + * v1.41.2 raised the cap to sixteen. */ -export const MAX_TURN_STEPS = 48; +export const MAX_TURN_STEPS = 64; const SCOPE_SOURCES: ReadonlySet = new Set( coachScopeSourceSchema.options, diff --git a/src/lib/ai/coach/types.ts b/src/lib/ai/coach/types.ts index 2db401956..686e6232f 100644 --- a/src/lib/ai/coach/types.ts +++ b/src/lib/ai/coach/types.ts @@ -498,7 +498,7 @@ export type CoachStepReason = | "invalid_arguments"; export interface CoachStep { - /** `s1`..`s48`, unique within a turn; frames upsert by id. */ + /** `s1`..`s64`, unique within a turn; frames upsert by id. */ id: string; tool: CoachToolName | "show_result" | "snapshot"; /** Closed catalog key, e.g. `coach.step.read`. */ diff --git a/src/lib/ai/codex-client.ts b/src/lib/ai/codex-client.ts index 8a4fe0fdf..2eb7f2eaf 100644 --- a/src/lib/ai/codex-client.ts +++ b/src/lib/ai/codex-client.ts @@ -1,4 +1,4 @@ -import { randomUUID } from "node:crypto"; +import { createHash, randomUUID } from "node:crypto"; import { callTimeoutMs } from "./effective-timeout"; import { safeFetch } from "@/lib/safe-fetch"; import type { @@ -431,6 +431,21 @@ export interface CodexAttemptDiagnostics { workingSlug: string | null; } +/** + * A UUID-shaped id derived from a cache key, for the session headers: the + * same key gives the same id, and the key itself never rides a header. + */ +function uuidFromKey(key: string): string { + const hex = createHash("sha256").update(`codex-session:${key}`).digest("hex"); + return [ + hex.slice(0, 8), + hex.slice(8, 12), + `4${hex.slice(13, 16)}`, + `${((parseInt(hex[16], 16) & 0x3) | 0x8).toString(16)}${hex.slice(17, 20)}`, + hex.slice(20, 32), + ].join("-"); +} + export class CodexClient implements AIProvider { readonly type = "codex" as const; /** See `AIProvider.responseTimeoutSeconds`; stamped by the resolver. */ @@ -650,8 +665,15 @@ export class CodexClient implements AIProvider { slug: string, wire: CodexReasoningWire, ): Promise { - const sessionId = randomUUID(); - const threadId = randomUUID(); + // v1.41.2 — calls that share a cache key share one session and thread + // id and send the key as `prompt_cache_key`, the way the official client + // sends its thread id (docs/codex-protocol-spec.md §3b/§3c). Before, every + // round of a Coach turn arrived as a new session with no key, so the + // backend had no reason to route it to the cache the previous round + // filled, and each round was charged its whole prefix again. + const cacheKey = params.cacheKey?.trim() || null; + const sessionId = cacheKey ? uuidFromKey(cacheKey) : randomUUID(); + const threadId = cacheKey ? sessionId : randomUUID(); // v1.18.11 — fold vision inputs (Lab-OCR) into the user turn when present. // The Codex/ChatGPT-OAuth backend accepts the `input_image` content block @@ -713,6 +735,7 @@ export class CodexClient implements AIProvider { : null, store: false, stream: true, + ...(cacheKey ? { prompt_cache_key: cacheKey } : {}), // Required field; empty array when not asking for reasoning. With // reasoning on, the encrypted items come back so the next tool // round can hand them over statelessly (`store` must stay false). diff --git a/src/lib/ai/local-client.ts b/src/lib/ai/local-client.ts index 3c3a5f184..d4ea0fbfb 100644 --- a/src/lib/ai/local-client.ts +++ b/src/lib/ai/local-client.ts @@ -11,6 +11,7 @@ import type { import { buildOpenAIMessages, mapFinishReason, + parseCachedTokens, raiseEmbeddedError, readEmbeddedError, sanitiseBodyExcerpt, @@ -411,6 +412,7 @@ export class LocalOpenAICompatibleClient implements AIProvider { }>; usage?: { total_tokens?: number; + prompt_tokens_details?: { cached_tokens?: number }; completion_tokens_details?: { reasoning_tokens?: number }; }; }; @@ -459,6 +461,11 @@ export class LocalOpenAICompatibleClient implements AIProvider { return { content, tokensUsed: json.usage?.total_tokens ?? null, + // v1.41.2 — llama.cpp, vLLM and the hosted gateways this client also + // serves report a prompt-cache hit the Chat Completions way; without it + // the Coach turn budget counted a cached prefix in full. Ollama sends + // none, which leaves it null. + cachedInputTokens: parseCachedTokens(json), model: this.config.model, providerType: "local", // v1.28.28 (#470) — surface why the model stopped so the JSON callers @@ -543,7 +550,10 @@ export class LocalOpenAICompatibleClient implements AIProvider { message?: { content?: string }; finish_reason?: string; }>; - usage?: { total_tokens?: number }; + usage?: { + total_tokens?: number; + prompt_tokens_details?: { cached_tokens?: number }; + }; } | null; // The server ignored `stream` and answered buffered — the embedded // failure can ride this body exactly as it rides the non-streaming one. @@ -577,6 +587,7 @@ export class LocalOpenAICompatibleClient implements AIProvider { return { content: buffered, tokensUsed: json?.usage?.total_tokens ?? null, + cachedInputTokens: json ? parseCachedTokens(json) : null, model: this.config.model, providerType: "local", finishReason: mapFinishReason(json?.choices?.[0]?.finish_reason), @@ -589,6 +600,7 @@ export class LocalOpenAICompatibleClient implements AIProvider { let sseBuffer = ""; let content = ""; let tokensUsed: number | null = null; + let cachedInputTokens: number | null = null; let reasoningTokens: number | null = null; let finishReason: string | undefined; @@ -679,6 +691,7 @@ export class LocalOpenAICompatibleClient implements AIProvider { }>; usage?: { total_tokens?: number; + prompt_tokens_details?: { cached_tokens?: number }; completion_tokens_details?: { reasoning_tokens?: number }; }; }; @@ -702,6 +715,8 @@ export class LocalOpenAICompatibleClient implements AIProvider { if (typeof chunk.usage?.total_tokens === "number") { tokensUsed = chunk.usage.total_tokens; } + const cached = chunk.usage?.prompt_tokens_details?.cached_tokens; + if (typeof cached === "number") cachedInputTokens = cached; const rt = chunk.usage?.completion_tokens_details?.reasoning_tokens; if (typeof rt === "number") reasoningTokens = rt; const chunkFinish = chunk.choices?.[0]?.finish_reason; @@ -761,6 +776,7 @@ export class LocalOpenAICompatibleClient implements AIProvider { return { content, tokensUsed, + cachedInputTokens, model: this.config.model, providerType: "local", finishReason: mapFinishReason(finishReason), diff --git a/src/lib/ai/openai-capabilities.ts b/src/lib/ai/openai-capabilities.ts index 75b723a8e..1766ec05d 100644 --- a/src/lib/ai/openai-capabilities.ts +++ b/src/lib/ai/openai-capabilities.ts @@ -34,7 +34,7 @@ function isModernCanonicalModel(model: string): boolean { ); } -function isCanonicalOpenAIEndpoint(baseUrl: string): boolean { +export function isCanonicalOpenAIEndpoint(baseUrl: string): boolean { try { const url = new URL(baseUrl); return ( diff --git a/src/lib/ai/openai-client.ts b/src/lib/ai/openai-client.ts index 227613d67..46c084158 100644 --- a/src/lib/ai/openai-client.ts +++ b/src/lib/ai/openai-client.ts @@ -3,6 +3,7 @@ import { callTimeoutMs } from "./effective-timeout"; import { annotate } from "@/lib/logging/context"; import type { AIProvider, CompletionParams, CompletionResult } from "./types"; import { + isCanonicalOpenAIEndpoint, selectOpenAIChatCompletionsCapabilities, type OpenAIChatCompletionsCapabilities, } from "./openai-capabilities"; @@ -208,6 +209,13 @@ export class OpenAIClient implements AIProvider { : {}), ...(tools ? { tools } : {}), ...(params.toolChoice ? { tool_choice: params.toolChoice } : {}), + // v1.41.2 — the prompt-cache routing key `api.openai.com` documents + // (https://platform.openai.com/docs/guides/prompt-caching). Only + // there: a gateway or proxy behind another base URL may refuse a + // field it does not know. + ...(params.cacheKey && isCanonicalOpenAIEndpoint(this.config.baseUrl) + ? { prompt_cache_key: params.cacheKey } + : {}), // #1126 — the gateway entry's reasoning setting; Default sends // nothing. `api.openai.com` and Codex never get it. v1.41 — a call // that carries its own resolved level (`params.reasoning`) wins. diff --git a/src/lib/ai/openai-wire.ts b/src/lib/ai/openai-wire.ts index 673c128a7..f7ebb3ff1 100644 --- a/src/lib/ai/openai-wire.ts +++ b/src/lib/ai/openai-wire.ts @@ -150,7 +150,9 @@ export function parseOpenAIToolCalls( })); } -export function parseCachedTokens(json: OpenAIResponseJson): number | null { +export function parseCachedTokens(json: { + usage?: { prompt_tokens_details?: { cached_tokens?: number } }; +}): number | null { return json.usage?.prompt_tokens_details?.cached_tokens ?? null; } diff --git a/src/lib/ai/types.ts b/src/lib/ai/types.ts index c33949131..be66d3c42 100644 --- a/src/lib/ai/types.ts +++ b/src/lib/ai/types.ts @@ -374,6 +374,16 @@ export interface CompletionParams { * client sends exactly what it sent before the field existed. */ reasoning?: { effort: ReasoningLevel; summaries: boolean }; + /** + * v1.41.2 — a stable key for calls that resend the same prompt prefix (the + * rounds of one Coach turn, the turns of one conversation). Providers that + * route a request to a prompt cache by key read it: Codex as + * `prompt_cache_key` and its session headers, `api.openai.com` as + * `prompt_cache_key`. Without it a Codex call is routed afresh every time + * and a later round pays for its whole prefix again. An opaque value: the + * caller hashes whatever it derives the key from. Absent → nothing is sent. + */ + cacheKey?: string; /** * v1.41 — live reasoning events, called as they arrive on clients that read * the provider's stream, or once after the reply on clients that do not. @@ -390,6 +400,12 @@ export interface CompletionResult { * from the provider's cache, where the provider reports it (Anthropic * `cache_read_input_tokens`, OpenAI/Codex `cached_tokens`). Null / absent * when the provider does not report it or the prefix did not hit. + * + * v1.41.2 — always a share of `tokensUsed`, never beside it: the turn + * budget and the daily ledger charge `tokensUsed - cachedInputTokens` at + * full weight and the cached share at a tenth. A client whose provider + * reports cached input outside its input count (Anthropic) adds it to + * `tokensUsed`. */ cachedInputTokens?: number | null; model: string;