Widen the Coach turn budget and keep its rounds on the prompt cache

Own-plan turns get 165k tokens, 200 s and 16 rounds, operator turns 55k,
120 s and 8; the final-answer reserve, the concurrent-turn slot and the
step cap move with them.

Codex requests carry a per-conversation prompt_cache_key and a stable
session id, api.openai.com gets the same key, Anthropic usage counts
cache reads and writes in the gross, and local servers report cached
prompt tokens, so the turn budget discounts cached input on every
provider.
This commit is contained in:
Marc-André Bombeck committed 2026-10-07 22:48:48 +02:00
1 parent dce53291a4
commit 316e52dee3
24 files changed
+632 -99

No files matched your search

@@ -29,7 +29,7 @@ exports[`coach chat golden transcripts > 01a tool mode, two rounds, present data
],
},
{
"amount": 12891,
"amount": 13341,
"call": "reserveBudget",
"cap": 2000000,
"dateKey": "2026-09-26",
@@ -78,7 +78,7 @@ exports[`coach chat golden transcripts > 01a tool mode, two rounds, present data
"cached": 0,
"call": "reconcileSpend",
"dateKey": "2026-09-26",
"reserved": 1836,
"reserved": 2286,
},
{
"call": "appendMessage",
@@ -278,7 +278,7 @@ exports[`coach chat golden transcripts > 01b tool mode, present data, one figure
],
},
{
"amount": 12891,
"amount": 13341,
"call": "reserveBudget",
"cap": 2000000,
"dateKey": "2026-09-26",
@@ -327,7 +327,7 @@ exports[`coach chat golden transcripts > 01b tool mode, present data, one figure
"cached": 0,
"call": "reconcileSpend",
"dateKey": "2026-09-26",
"reserved": 1836,
"reserved": 2286,
},
{
"call": "annotate",
@@ -539,7 +539,7 @@ exports[`coach chat golden transcripts > 02 tool mode, every tool missed, fabric
],
},
{
"amount": 12889,
"amount": 13339,
"call": "reserveBudget",
"cap": 2000000,
"dateKey": "2026-09-26",
@@ -588,7 +588,7 @@ exports[`coach chat golden transcripts > 02 tool mode, every tool missed, fabric
"cached": 0,
"call": "reconcileSpend",
"dateKey": "2026-09-26",
"reserved": 1836,
"reserved": 2286,
},
{
"call": "annotate",
@@ -1415,7 +1415,7 @@ exports[`coach chat golden transcripts > 07a all providers failed (plain) 1`] =
],
},
{
"amount": 12891,
"amount": 13341,
"call": "reserveBudget",
"cap": 2000000,
"dateKey": "2026-09-26",
@@ -1453,7 +1453,7 @@ exports[`coach chat golden transcripts > 07a all providers failed (plain) 1`] =
"cached": 0,
"call": "reconcileSpend",
"dateKey": "2026-09-26",
"reserved": 12891,
"reserved": 13341,
},
{
"call": "annotate",
@@ -1507,7 +1507,7 @@ exports[`coach chat golden transcripts > 07b all providers failed (every hop rat
],
},
{
"amount": 12891,
"amount": 13341,
"call": "reserveBudget",
"cap": 2000000,
"dateKey": "2026-09-26",
@@ -1545,7 +1545,7 @@ exports[`coach chat golden transcripts > 07b all providers failed (every hop rat
"cached": 0,
"call": "reconcileSpend",
"dateKey": "2026-09-26",
"reserved": 12891,
"reserved": 13341,
},
{
"call": "annotate",
@@ -1599,7 +1599,7 @@ exports[`coach chat golden transcripts > 07c all providers failed (primary crede
],
},
{
"amount": 12891,
"amount": 13341,
"call": "reserveBudget",
"cap": 2000000,
"dateKey": "2026-09-26",
@@ -1637,7 +1637,7 @@ exports[`coach chat golden transcripts > 07c all providers failed (primary crede
"cached": 0,
"call": "reconcileSpend",
"dateKey": "2026-09-26",
"reserved": 12891,
"reserved": 13341,
},
{
"call": "annotate",
@@ -1691,7 +1691,7 @@ exports[`coach chat golden transcripts > 08 an unwrapped tagged provider error 1
],
},
{
"amount": 12891,
"amount": 13341,
"call": "reserveBudget",
"cap": 2000000,
"dateKey": "2026-09-26",
@@ -1729,7 +1729,7 @@ exports[`coach chat golden transcripts > 08 an unwrapped tagged provider error 1
"cached": 0,
"call": "reconcileSpend",
"dateKey": "2026-09-26",
"reserved": 12891,
"reserved": 13341,
},
{
"call": "annotate",
@@ -1784,7 +1784,7 @@ exports[`coach chat golden transcripts > 09 an aborted request leaves a cancelle
],
},
{
"amount": 12891,
"amount": 13341,
"call": "reserveBudget",
"cap": 2000000,
"dateKey": "2026-09-26",
@@ -1822,7 +1822,7 @@ exports[`coach chat golden transcripts > 09 an aborted request leaves a cancelle
"cached": 0,
"call": "reconcileSpend",
"dateKey": "2026-09-26",
"reserved": 12891,
"reserved": 13341,
},
{
"call": "appendMessage",
@@ -1883,7 +1883,7 @@ exports[`coach chat golden transcripts > 10 outbound block replaces the reply an
],
},
{
"amount": 12892,
"amount": 13342,
"call": "reserveBudget",
"cap": 2000000,
"dateKey": "2026-09-26",
@@ -1932,7 +1932,7 @@ exports[`coach chat golden transcripts > 10 outbound block replaces the reply an
"cached": 0,
"call": "reconcileSpend",
"dateKey": "2026-09-26",
"reserved": 1836,
"reserved": 2286,
},
{
"call": "annotate",
@@ -2262,7 +2262,7 @@ exports[`coach chat golden transcripts > 11a KEYVALUES valid 1`] = `
],
},
{
"amount": 12891,
"amount": 13341,
"call": "reserveBudget",
"cap": 2000000,
"dateKey": "2026-09-26",
@@ -2311,7 +2311,7 @@ exports[`coach chat golden transcripts > 11a KEYVALUES valid 1`] = `
"cached": 0,
"call": "reconcileSpend",
"dateKey": "2026-09-26",
"reserved": 1836,
"reserved": 2286,
},
{
"call": "appendMessage",
@@ -2515,7 +2515,7 @@ exports[`coach chat golden transcripts > 11b KEYVALUES partial 1`] = `
],
},
{
"amount": 12891,
"amount": 13341,
"call": "reserveBudget",
"cap": 2000000,
"dateKey": "2026-09-26",
@@ -2564,7 +2564,7 @@ exports[`coach chat golden transcripts > 11b KEYVALUES partial 1`] = `
"cached": 0,
"call": "reconcileSpend",
"dateKey": "2026-09-26",
"reserved": 1836,
"reserved": 2286,
},
{
"call": "annotate",
@@ -2766,7 +2766,7 @@ exports[`coach chat golden transcripts > 11c KEYVALUES malformed (no closing mar
],
},
{
"amount": 12891,
"amount": 13341,
"call": "reserveBudget",
"cap": 2000000,
"dateKey": "2026-09-26",
@@ -2815,7 +2815,7 @@ exports[`coach chat golden transcripts > 11c KEYVALUES malformed (no closing mar
"cached": 0,
"call": "reconcileSpend",
"dateKey": "2026-09-26",
"reserved": 1836,
"reserved": 2286,
},
{
"call": "annotate",
@@ -3001,7 +3001,7 @@ exports[`coach chat golden transcripts > 11d sentinel-only reply is an empty rep
],
},
{
"amount": 12891,
"amount": 13341,
"call": "reserveBudget",
"cap": 2000000,
"dateKey": "2026-09-26",
@@ -3050,7 +3050,7 @@ exports[`coach chat golden transcripts > 11d sentinel-only reply is an empty rep
"cached": 0,
"call": "reconcileSpend",
"dateKey": "2026-09-26",
"reserved": 1836,
"reserved": 2286,
},
{
"call": "annotate",
@@ -3104,7 +3104,7 @@ exports[`coach chat golden transcripts > 11e empty provider reply 1`] = `
],
},
{
"amount": 12891,
"amount": 13341,
"call": "reserveBudget",
"cap": 2000000,
"dateKey": "2026-09-26",
@@ -3153,7 +3153,7 @@ exports[`coach chat golden transcripts > 11e empty provider reply 1`] = `
"cached": 0,
"call": "reconcileSpend",
"dateKey": "2026-09-26",
"reserved": 1836,
"reserved": 2286,
},
],
"contentType": "text/event-stream; charset=utf-8",
@@ -3198,7 +3198,7 @@ exports[`coach chat golden transcripts > 12a SUGGEST-REMINDER surfaced 1`] = `
],
},
{
"amount": 12896,
"amount": 13346,
"call": "reserveBudget",
"cap": 2000000,
"dateKey": "2026-09-26",
@@ -3247,7 +3247,7 @@ exports[`coach chat golden transcripts > 12a SUGGEST-REMINDER surfaced 1`] = `
"cached": 0,
"call": "reconcileSpend",
"dateKey": "2026-09-26",
"reserved": 1837,
"reserved": 2287,
},
{
"call": "prisma.user.update",
@@ -3471,7 +3471,7 @@ exports[`coach chat golden transcripts > 12b SUGGEST-REMINDER suppressed 1`] = `
],
},
{
"amount": 12896,
"amount": 13346,
"call": "reserveBudget",
"cap": 2000000,
"dateKey": "2026-09-26",
@@ -3520,7 +3520,7 @@ exports[`coach chat golden transcripts > 12b SUGGEST-REMINDER suppressed 1`] = `
"cached": 0,
"call": "reconcileSpend",
"dateKey": "2026-09-26",
"reserved": 1837,
"reserved": 2287,
},
{
"call": "annotate",
@@ -3708,7 +3708,7 @@ exports[`coach chat golden transcripts > 13 SUGGEST-ACTION 1`] = `
],
},
{
"amount": 12892,
"amount": 13342,
"call": "reserveBudget",
"cap": 2000000,
"dateKey": "2026-09-26",
@@ -3757,7 +3757,7 @@ exports[`coach chat golden transcripts > 13 SUGGEST-ACTION 1`] = `
"cached": 0,
"call": "reconcileSpend",
"dateKey": "2026-09-26",
"reserved": 1836,
"reserved": 2286,
},
{
"call": "annotate",
@@ -3985,7 +3985,7 @@ exports[`coach chat golden transcripts > 14a REMEMBER valid 1`] = `
],
},
{
"amount": 12896,
"amount": 13346,
"call": "reserveBudget",
"cap": 2000000,
"dateKey": "2026-09-26",
@@ -4034,7 +4034,7 @@ exports[`coach chat golden transcripts > 14a REMEMBER valid 1`] = `
"cached": 0,
"call": "reconcileSpend",
"dateKey": "2026-09-26",
"reserved": 1837,
"reserved": 2287,
},
{
"call": "captureReminderFromSentinel",
@@ -4242,7 +4242,7 @@ exports[`coach chat golden transcripts > 14b REMEMBER malformed 1`] = `
],
},
{
"amount": 12896,
"amount": 13346,
"call": "reserveBudget",
"cap": 2000000,
"dateKey": "2026-09-26",
@@ -4291,7 +4291,7 @@ exports[`coach chat golden transcripts > 14b REMEMBER malformed 1`] = `
"cached": 0,
"call": "reconcileSpend",
"dateKey": "2026-09-26",
"reserved": 1837,
"reserved": 2287,
},
{
"call": "annotate",
@@ -4494,7 +4494,7 @@ exports[`coach chat golden transcripts > 15 an unknown learn link is scrubbed, a
],
},
{
"amount": 12895,
"amount": 13345,
"call": "reserveBudget",
"cap": 2000000,
"dateKey": "2026-09-26",
@@ -4543,7 +4543,7 @@ exports[`coach chat golden transcripts > 15 an unknown learn link is scrubbed, a
"cached": 0,
"call": "reconcileSpend",
"dateKey": "2026-09-26",
"reserved": 1837,
"reserved": 2287,
},
{
"call": "annotate",
@@ -5207,7 +5207,7 @@ exports[`coach chat golden transcripts > 17a a workout-scoped first turn 1`] = `
],
},
{
"amount": 12890,
"amount": 13340,
"call": "reserveBudget",
"cap": 2000000,
"dateKey": "2026-09-26",
@@ -5256,7 +5256,7 @@ exports[`coach chat golden transcripts > 17a a workout-scoped first turn 1`] = `
"cached": 0,
"call": "reconcileSpend",
"dateKey": "2026-09-26",
"reserved": 1836,
"reserved": 2286,
},
{
"call": "appendMessage",
@@ -5465,7 +5465,7 @@ exports[`coach chat golden transcripts > 17b a workout-scoped first turn with th
],
},
{
"amount": 12890,
"amount": 13340,
"call": "reserveBudget",
"cap": 2000000,
"dateKey": "2026-09-26",
@@ -5514,7 +5514,7 @@ exports[`coach chat golden transcripts > 17b a workout-scoped first turn with th
"cached": 0,
"call": "reconcileSpend",
"dateKey": "2026-09-26",
"reserved": 1836,
"reserved": 2286,
},
{
"call": "appendMessage",
@@ -5702,7 +5702,7 @@ exports[`coach chat golden transcripts > 18 a persistence throw after the provid
],
},
{
"amount": 12891,
"amount": 13341,
"call": "reserveBudget",
"cap": 2000000,
"dateKey": "2026-09-26",
@@ -5751,7 +5751,7 @@ exports[`coach chat golden transcripts > 18 a persistence throw after the provid
"cached": 0,
"call": "reconcileSpend",
"dateKey": "2026-09-26",
"reserved": 1836,
"reserved": 2286,
},
{
"call": "event.setError",
@@ -5817,7 +5817,7 @@ exports[`coach chat golden transcripts > 19 an existing conversation with prior
],
},
{
"amount": 13053,
"amount": 13503,
"call": "reserveBudget",
"cap": 2000000,
"dateKey": "2026-09-26",
@@ -5866,7 +5866,7 @@ exports[`coach chat golden transcripts > 19 an existing conversation with prior
"cached": 0,
"call": "reconcileSpend",
"dateKey": "2026-09-26",
"reserved": 1851,
"reserved": 2301,
},
{
"call": "annotate",
@@ -6078,7 +6078,7 @@ exports[`coach chat golden transcripts > 20 a guided-question turn with a scope
],
},
{
"amount": 12892,
"amount": 13342,
"call": "reserveBudget",
"cap": 2000000,
"dateKey": "2026-09-26",
@@ -6127,7 +6127,7 @@ exports[`coach chat golden transcripts > 20 a guided-question turn with a scope
"cached": 0,
"call": "reconcileSpend",
"dateKey": "2026-09-26",
"reserved": 1836,
"reserved": 2286,
},
{
"call": "appendMessage",
@@ -6295,7 +6295,7 @@ exports[`coach chat golden transcripts > 21 the German locale with a stored self
],
},
{
"amount": 12917,
"amount": 13367,
"call": "reserveBudget",
"cap": 2000000,
"dateKey": "2026-09-26",
@@ -6344,7 +6344,7 @@ exports[`coach chat golden transcripts > 21 the German locale with a stored self
"cached": 0,
"call": "reconcileSpend",
"dateKey": "2026-09-26",
"reserved": 1839,
"reserved": 2289,
},
{
"call": "appendMessage",
@@ -6548,7 +6548,7 @@ exports[`coach chat golden transcripts > 22 tool mode with live steps: found, em
],
},
{
"amount": 12891,
"amount": 13341,
"call": "reserveBudget",
"cap": 2000000,
"dateKey": "2026-09-26",
@@ -6597,7 +6597,7 @@ exports[`coach chat golden transcripts > 22 tool mode with live steps: found, em
"cached": 0,
"call": "reconcileSpend",
"dateKey": "2026-09-26",
"reserved": 1836,
"reserved": 2286,
},
{
"call": "appendMessage",
@@ -63,6 +63,7 @@ export interface ProviderCall {
system: string;
messages: Array<{ role: string; content: string }>;
toolChoice: string | undefined;
cacheKey: string | undefined;
toolCalls: Array<{ name: string; args: Record<string, unknown> }>;
}
export const providerCalls: ProviderCall[] = [];
@@ -278,6 +279,7 @@ async function runRawCompletionWithFallback(args: {
system: string;
messages: Array<{ role: string; content: string }>;
toolChoice?: string;
cacheKey?: string;
};
}) {
const round = world.script.shift() ?? { text: "" };
@@ -289,6 +291,7 @@ async function runRawCompletionWithFallback(args: {
content: msg.content,
})),
toolChoice: args.params.toolChoice,
cacheKey: args.params.cacheKey,
toolCalls: calls,
});
const offered = args.params.toolChoice !== "none";
@@ -279,7 +279,9 @@ describe("concurrent turns", () => {
expect(m.checkRateLimit).toHaveBeenCalledWith(
"coach-turn-active:u1",
2,
180_000,
// The longest turn's wall time (200 s since v1.41.2) plus 30 s for
// its reply to stream.
230_000,
);
});
});
+6 -4
View File
@@ -40,6 +40,7 @@ import { runCoachTurn } from "@/lib/ai/coach/turn/pipeline";
import type { ReasoningLevel } from "@/lib/ai/reasoning/levels";
import { resolveCoachTurnReasoningLevel } from "@/lib/ai/reasoning/controls";
import { streamProviderError, streamRefusal } from "@/lib/ai/coach/turn/sse";
import { TURN_LIMITS } from "@/lib/ai/coach/tools/turn-budget";
/**
* The `coach` capability gate for a turn. Every unavailable reason throws the
@@ -72,11 +73,12 @@ async function coachCapabilityRefusal(): Promise<Response | null> {
*/
const COACH_CONCURRENT_TURNS = 2;
/**
* How long a turn's slot is held at most: a turn's wall time (150 s) plus
* the streaming of its reply. A slot whose release never arrived (a crashed
* process) frees itself when the window ends.
* How long a turn's slot is held at most: the longest turn's wall time
* (`TURN_LIMITS.user.wallMs`, 200 s) plus 30 s for the streaming of its
* reply. A slot whose release never arrived (a crashed process) frees itself
* when the window ends.
*/
const COACH_TURN_SLOT_MS = 180_000;
const COACH_TURN_SLOT_MS = TURN_LIMITS.user.wallMs + 30_000;
async function handleChatRequest(request: NextRequest): Promise<Response> {
const auth = await requireAuth();
@@ -0,0 +1,368 @@
/**
* Cached input, per provider, end to end into the Coach turn budget.
*
* The turn budget counts cached input at a tenth (`CACHED_INPUT_WEIGHT`).
* That only works when two things hold for every client:
*
* 1. `tokensUsed` is the gross count, cached input included, and
* `cachedInputTokens` is the cached share of it. The budget and the
* daily ledger both subtract the cached share from the gross.
* 2. The cached count is read from where the provider puts it: Codex and
* the Responses wire under `usage.input_tokens_details.cached_tokens`,
* Chat Completions (OpenAI, gateways, local servers that report it)
* under `usage.prompt_tokens_details.cached_tokens`, Anthropic as
* `usage.cache_read_input_tokens` beside an `input_tokens` that does
* NOT include it.
*
* Each fixture below is a usage payload in the provider's own shape for the
* same round: 20,000 tokens of input of which 18,000 came from the cache,
* and 500 of output. Every client must turn it into gross 20,500 / cached
* 18,000, which the budget weighs at 2,000 + 500 + 1,800 = 4,300.
*
* And the cache has to be hit at all: the Codex backend routes a request to
* a cache by `prompt_cache_key` (the official client sets it to its thread
* id), and `api.openai.com` documents the same field. A turn's rounds share
* one key, so round two finds round one's prefix.
*/
import { beforeEach, describe, expect, it, vi } from "vitest";
vi.mock("undici", async (importOriginal) => {
const actual = await importOriginal<typeof import("undici")>();
return {
...actual,
fetch: (input: unknown, init?: unknown) =>
(globalThis.fetch as unknown as (i: unknown, n?: unknown) => unknown)(
input,
init,
),
};
});
vi.mock("../local-host-allowlist", () => ({ aiEgressPolicyFor: () => ({}) }));
import { AnthropicClient } from "../anthropic-client";
import { CodexClient } from "../codex-client";
import { OpenAIClient } from "../openai-client";
import { LocalOpenAICompatibleClient } from "../local-client";
import { singleUserTurn, type CompletionResult } from "../types";
import { weightedRoundTokens } from "@/lib/ai/coach/tools/turn-budget";
const INPUT = 20_000;
const CACHED = 18_000;
const OUTPUT = 500;
const GROSS = INPUT + OUTPUT;
const WEIGHTED = INPUT - CACHED + OUTPUT + CACHED / 10;
function expectDiscounted(result: CompletionResult) {
expect(result.tokensUsed).toBe(GROSS);
expect(result.cachedInputTokens).toBe(CACHED);
expect(
weightedRoundTokens({
tokens: result.tokensUsed,
cachedTokens: result.cachedInputTokens,
}),
).toBe(WEIGHTED);
}
function sse(events: string[]): Response {
const encoder = new TextEncoder();
return new Response(
new ReadableStream<Uint8Array>({
start(controller) {
for (const ev of events) controller.enqueue(encoder.encode(ev));
controller.close();
},
}),
{ status: 200, headers: { "content-type": "text/event-stream" } },
);
}
function codexStream(): Response {
return sse([
`event: response.output_item.done\ndata: ${JSON.stringify({
type: "response.output_item.done",
item: {
type: "message",
role: "assistant",
content: [{ type: "output_text", text: "ok" }],
},
})}\n\n`,
`event: response.completed\ndata: ${JSON.stringify({
type: "response.completed",
response: {
id: "resp1",
usage: {
input_tokens: INPUT,
input_tokens_details: { cached_tokens: CACHED },
output_tokens: OUTPUT,
output_tokens_details: { reasoning_tokens: 0 },
total_tokens: GROSS,
},
},
})}\n\n`,
]);
}
/** The Chat Completions usage block, as OpenAI and most gateways send it. */
const chatUsage = {
prompt_tokens: INPUT,
completion_tokens: OUTPUT,
total_tokens: GROSS,
prompt_tokens_details: { cached_tokens: CACHED },
};
function jsonResponse(body: unknown) {
return {
ok: true,
status: 200,
headers: { get: () => "application/json" },
json: () => Promise.resolve(body),
text: () => Promise.resolve(JSON.stringify(body)),
};
}
function codexClient() {
return new CodexClient({
accessToken: "t",
accountId: "a",
onTokenRefresh: vi
.fn()
.mockResolvedValue({ accessToken: "t", accountId: "a" }),
slugChain: ["gpt-5-codex"],
});
}
beforeEach(() => {
vi.restoreAllMocks();
});
describe("cached input reaches the turn budget, per provider", () => {
it("codex: input_tokens_details.cached_tokens", async () => {
vi.stubGlobal("fetch", vi.fn().mockResolvedValue(codexStream()));
const result = await codexClient().generateCompletion(
singleUserTurn({ system: "s", user: "u" }),
);
expectDiscounted(result);
});
it("openai (Chat Completions): prompt_tokens_details.cached_tokens", async () => {
vi.stubGlobal(
"fetch",
vi.fn().mockResolvedValue(
jsonResponse({
choices: [{ message: { content: "ok" }, finish_reason: "stop" }],
usage: chatUsage,
}),
),
);
const client = new OpenAIClient({
apiKey: "sk-test",
model: "gpt-4o-mini",
baseUrl: "https://api.openai.com/v1",
});
expectDiscounted(
await client.generateCompletion(
singleUserTurn({ system: "s", user: "u" }),
),
);
});
it("gateway (OpenRouter and the like): the same Chat Completions block", async () => {
vi.stubGlobal(
"fetch",
vi.fn().mockResolvedValue(
jsonResponse({
choices: [{ message: { content: "ok" }, finish_reason: "stop" }],
usage: chatUsage,
}),
),
);
const client = new OpenAIClient({
apiKey: "sk-or-test",
model: "openai/gpt-4o-mini",
baseUrl: "https://openrouter.ai/api/v1",
providerType: "openai-compatible",
});
expectDiscounted(
await client.generateCompletion(
singleUserTurn({ system: "s", user: "u" }),
),
);
});
it("anthropic: cache reads ride beside input_tokens, not inside it", async () => {
vi.stubGlobal(
"fetch",
vi.fn().mockResolvedValue(
jsonResponse({
content: [{ type: "text", text: "ok" }],
stop_reason: "end_turn",
usage: {
// Anthropic's input_tokens counts only what came after the last
// cache breakpoint; reads and writes are reported beside it.
input_tokens: INPUT - CACHED,
cache_read_input_tokens: CACHED,
cache_creation_input_tokens: 0,
output_tokens: OUTPUT,
},
}),
),
);
const client = new AnthropicClient({
apiKey: "sk-ant-test",
model: "claude-sonnet-4-5",
});
expectDiscounted(
await client.generateCompletion(
singleUserTurn({ system: "s", user: "u" }),
),
);
});
it("anthropic: a cache write counts in full", async () => {
vi.stubGlobal(
"fetch",
vi.fn().mockResolvedValue(
jsonResponse({
content: [{ type: "text", text: "ok" }],
stop_reason: "end_turn",
usage: {
input_tokens: 2_000,
cache_read_input_tokens: 0,
cache_creation_input_tokens: 18_000,
output_tokens: OUTPUT,
},
}),
),
);
const client = new AnthropicClient({
apiKey: "sk-ant-test",
model: "claude-sonnet-4-5",
});
const result = await client.generateCompletion(
singleUserTurn({ system: "s", user: "u" }),
);
expect(result.tokensUsed).toBe(GROSS);
expect(result.cachedInputTokens ?? 0).toBe(0);
});
it("local: prompt_tokens_details.cached_tokens where the server reports it", async () => {
vi.stubGlobal(
"fetch",
vi.fn().mockResolvedValue(
jsonResponse({
choices: [{ message: { content: "ok" }, finish_reason: "stop" }],
usage: chatUsage,
}),
),
);
const client = new LocalOpenAICompatibleClient({
apiKey: null,
model: "llama3:8b",
baseUrl: "http://localhost:11434/v1",
});
expectDiscounted(
await client.generateCompletion(
singleUserTurn({ system: "s", user: "u" }),
),
);
});
it("local, streamed: the usage frame's cached count", async () => {
vi.stubGlobal(
"fetch",
vi
.fn()
.mockResolvedValue(
sse([
'data: {"choices":[{"delta":{"content":"ok"}}]}\n\n',
`data: ${JSON.stringify({ choices: [{ delta: {}, finish_reason: "stop" }], usage: chatUsage })}\n\n`,
"data: [DONE]\n\n",
]),
),
);
const client = new LocalOpenAICompatibleClient({
apiKey: null,
model: "llama3:8b",
baseUrl: "http://localhost:11434/v1",
});
expectDiscounted(
await client.generateCompletionStream(
singleUserTurn({ system: "s", user: "u" }),
() => {},
),
);
});
});
describe("a turn's rounds share one prompt cache key", () => {
it("codex sends prompt_cache_key and one session id for every round with the same key", async () => {
const fetchMock = vi
.fn()
.mockImplementation(() => Promise.resolve(codexStream()));
vi.stubGlobal("fetch", fetchMock);
const client = codexClient();
const params = {
...singleUserTurn({ system: "s", user: "u" }),
cacheKey: "coach-turn-key",
};
await client.generateCompletion(params);
await client.generateCompletion(params);
const [first, second] = fetchMock.mock.calls.map(
(call) => call[1] as { body: string; headers: Record<string, string> },
);
expect(JSON.parse(first.body).prompt_cache_key).toBe("coach-turn-key");
expect(JSON.parse(second.body).prompt_cache_key).toBe("coach-turn-key");
expect(first.headers.session_id).toBe(second.headers.session_id);
expect(first.headers.session_id).toMatch(
/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/,
);
});
it("codex without a key sends none and keeps a fresh session per call", async () => {
const fetchMock = vi
.fn()
.mockImplementation(() => Promise.resolve(codexStream()));
vi.stubGlobal("fetch", fetchMock);
const client = codexClient();
await client.generateCompletion(singleUserTurn({ system: "s", user: "u" }));
await client.generateCompletion(singleUserTurn({ system: "s", user: "u" }));
const [first, second] = fetchMock.mock.calls.map(
(call) => call[1] as { body: string; headers: Record<string, string> },
);
expect("prompt_cache_key" in JSON.parse(first.body)).toBe(false);
expect(first.headers.session_id).not.toBe(second.headers.session_id);
});
it("api.openai.com gets prompt_cache_key; a gateway never does", async () => {
const reply = () =>
Promise.resolve(
jsonResponse({
choices: [{ message: { content: "ok" }, finish_reason: "stop" }],
usage: chatUsage,
}),
);
const fetchMock = vi.fn().mockImplementation(reply);
vi.stubGlobal("fetch", fetchMock);
const params = {
...singleUserTurn({ system: "s", user: "u" }),
cacheKey: "coach-turn-key",
};
await new OpenAIClient({
apiKey: "sk-test",
model: "gpt-4o-mini",
baseUrl: "https://api.openai.com/v1",
}).generateCompletion(params);
await new OpenAIClient({
apiKey: "sk-or-test",
model: "openai/gpt-4o-mini",
baseUrl: "https://openrouter.ai/api/v1",
providerType: "openai-compatible",
}).generateCompletion(params);
const [direct, gateway] = fetchMock.mock.calls.map((call) =>
JSON.parse((call[1] as { body: string }).body),
);
expect(direct.prompt_cache_key).toBe("coach-turn-key");
expect("prompt_cache_key" in gateway).toBe(false);
});
});
+12 -1
View File
@@ -412,6 +412,7 @@ export class AnthropicClient implements AIProvider {
input_tokens?: number;
output_tokens?: number;
cache_read_input_tokens?: number;
cache_creation_input_tokens?: number;
output_tokens_details?: { thinking_tokens?: number };
};
};
@@ -487,9 +488,19 @@ export class AnthropicClient implements AIProvider {
const content =
wantsJson && rawText ? extractJsonObject(rawText) : (rawText ?? "");
// v1.41.2 — the gross count includes the cached prefix. Anthropic reports
// cache reads and cache writes BESIDE `input_tokens`, which holds only
// what came after the last breakpoint; every other client reports a total
// that already contains the cached share. The turn budget and the daily
// ledger subtract `cachedInputTokens` from `tokensUsed`, so leaving the
// reads out of the gross made a mostly cached round look nearly free and
// a cache write (billed above the base rate) went uncounted.
const inputTokens = json.usage?.input_tokens ?? 0;
const cacheReadTokens = json.usage?.cache_read_input_tokens ?? 0;
const cacheWriteTokens = json.usage?.cache_creation_input_tokens ?? 0;
const outputTokens = json.usage?.output_tokens ?? 0;
const tokensUsed = inputTokens + outputTokens || null;
const tokensUsed =
inputTokens + cacheReadTokens + cacheWriteTokens + outputTokens || null;
const cachedInputTokens = json.usage?.cache_read_input_tokens ?? null;
// The presence of tool_use blocks is authoritative for the tool-loop gate:
@@ -5,6 +5,7 @@
* chat turn's own context, the reply through the turn's parsers) and the
* grade, so a live run measures the model and nothing else.
*/
import { TURN_LIMITS } from "@/lib/ai/coach/tools/turn-budget";
import { afterEach, describe, expect, it, vi } from "vitest";
import type { CompletionParams, CompletionResult } from "@/lib/ai/types";
@@ -119,23 +120,27 @@ describe("runScenarioLive", () => {
it("forces an answer at the round cap of the person's own plan", async () => {
const call = toolRound("get_metric_table", { metric: "pulse" });
// v1.41.2 — the cap is sixteen (was twelve); read from the limits so
// the scenario run and the turn cannot drift apart again.
const cap = TURN_LIMITS.user.maxRounds;
expect(cap).toBe(16);
const provider = scripted([
...Array.from({ length: 11 }, () => call),
...Array.from({ length: cap - 1 }, () => call),
{ content: "So far." },
]);
const observation = await runScenarioLive({
scenario: scenario("forced-final.en"),
provider,
});
expect(observation.providerCalls).toBe(12);
expect(observation.providerCalls).toBe(cap);
expect(provider.sent.map((p) => p.toolChoice)).toEqual([
...Array.from({ length: 11 }, () => "auto"),
...Array.from({ length: cap - 1 }, () => "auto"),
"none",
]);
expect(observation.prose).toBe("So far.");
expect(observation.toolRounds).toBe(11);
expect(observation.toolRounds).toBe(cap - 1);
// The same call every round: graded as repeats.
expect(observation.repeatedCalls).toBe(10);
expect(observation.repeatedCalls).toBe(cap - 2);
});
it("ends the run on a clarifying question asked through the tool", async () => {
@@ -63,6 +63,7 @@ import type {
CoachStep,
} from "@/lib/ai/coach/types";
import { COACH_FOLLOW_UP_KEYS } from "@/lib/ai/coach/dialog-keys";
import { TURN_LIMITS } from "@/lib/ai/coach/tools/turn-budget";
import { POST } from "@/app/api/insights/chat/route";
import {
@@ -130,7 +131,7 @@ const call = (name: string, args: Record<string, unknown> = {}) => ({
});
/** Ten distinct reads: with `get_sleep` first, eleven fetching rounds. */
const FORCED_READS: ReadonlyArray<[string, string]> = [
const FORCED_READS: ReadonlyArray<[string, string, string?]> = [
["pulse", "last7days"],
["pulse", "last30days"],
["pulse", "last90days"],
@@ -141,6 +142,11 @@ const FORCED_READS: ReadonlyArray<[string, string]> = [
["bp", "lastYear"],
["sleep", "last90days"],
["sleep", "lastYear"],
// v1.41.2 — four more for the sixteen-round cap.
["sleep", "last7days"],
["pulse", "last30days", "previous"],
["sleep", "last30days"],
["bp", "last30days", "previous"],
];
function script(name: string, lang: Lang): Round[] {
@@ -204,12 +210,18 @@ function script(name: string, lang: Lang): Round[] {
return [];
case "forced-final":
// v1.41 — a new read every round, never a repeat, until the round cap
// of the person's own plan (twelve, the forced answer included) makes
// the twelfth round the answer.
// of the person's own plan (sixteen since v1.41.2, the forced answer
// included) makes the last round the answer.
return [
{ calls: [call("get_sleep")] },
...FORCED_READS.map(([metric, window]) => ({
calls: [call("get_metric_table", { metric, window })],
...FORCED_READS.map(([metric, window, period]) => ({
calls: [
call("get_metric_table", {
metric,
window,
...(period ? { period } : {}),
}),
],
})),
{ text: PROSE.forced[lang] },
];
@@ -766,9 +778,14 @@ describe.each(["de", "en"] as const)("in %s", (lang) => {
const scenario = byName("forced-final", lang);
const { observation, frames } = await run(scenario);
expect(providerCalls.map((c) => c.toolChoice)).toEqual([
...Array.from({ length: 11 }, () => "auto"),
...Array.from({ length: TURN_LIMITS.user.maxRounds - 1 }, () => "auto"),
"none",
]);
// v1.41.2 — every round of the turn carries the same prompt-cache key,
// so a provider that routes by key finds the prefix the last round left.
const keys = new Set(providerCalls.map((c) => c.cacheKey));
expect(keys.size).toBe(1);
expect([...keys][0]).toMatch(/^[0-9a-f]{32}$/);
const provenance = framesOf<{ metricSource: { forcedFinal?: boolean } }>(
frames,
"provenance",
+3 -2
View File
@@ -26,6 +26,7 @@
*/
import type { Locale } from "@/lib/i18n/config";
import { UNBOUNDED_REACH } from "@/lib/ai/coach/history-reach";
import { TURN_LIMITS } from "@/lib/ai/coach/tools/turn-budget";
import type {
AiMessage,
AiToolCall,
@@ -145,7 +146,7 @@ export interface ScenarioProvider {
* How many rounds a live scenario may take: the round cap of the person's
* own plan (v1.41), the forced answer included.
*/
const SCENARIO_MAX_ROUNDS = 12;
const SCENARIO_MAX_ROUNDS = TURN_LIMITS.user.maxRounds;
function parseArgs(raw: string): Record<string, unknown> {
try {
@@ -236,7 +237,7 @@ function priorResultTurns(scenario: CoachScenario) {
* the chat turn's own context (system prompt, tool-mode and dialog
* addenda, DATA INVENTORY, EARLIER TABLES, transcript), put to a real
* model over the real tool catalogue, comparisons and the dialog tools
* for up to twelve rounds (v1.41), a question ending the run. Every call is
* for as many rounds as a turn on the person's own plan (v1.41), a question ending the run. Every call is
* captured with its arguments and answered with a stand-in result; the
* reply goes through the clarification and chip parsers the turn uses.
*
+20
View File
@@ -0,0 +1,20 @@
/**
* v1.41.2 — the prompt-cache key of a Coach conversation.
*
* Every round of a turn resends the same system prompt and history, and the
* next turn of the conversation resends it again with one more exchange. A
* provider that routes requests to its prompt cache by key (Codex,
* `api.openai.com`) only finds the prefix an earlier call left there when the
* calls carry the same key. One key per conversation does that; a turn with
* no conversation yet gets a key of its own, still shared by its rounds.
*
* Hashed so the provider never sees the conversation id itself.
*/
import { createHash, randomUUID } from "node:crypto";
export function coachPromptCacheKey(conversationId?: string | null): string {
return createHash("sha256")
.update(`coach-prompt:${conversationId || randomUUID()}`)
.digest("hex")
.slice(0, 32);
}
+1 -1
View File
@@ -76,7 +76,7 @@ export const coachStepSchema = z
id: z
.string()
.regex(/^s[1-9]\d{0,2}$/)
.describe("`s1`..`s48`, unique within a turn."),
.describe("`s1`..`s64`, unique within a turn."),
tool: z.enum([...COACH_TOOL_NAMES, "show_result", "snapshot"]),
labelKey: z
.string()
@@ -25,18 +25,40 @@ function clock(start = 0) {
}
describe("TURN_LIMITS", () => {
// v1.41.2 — raised by about a third from 120k / 150 s / 12 and
// 40k / 90 s / 6: a turn on a person's own plan stopped for tokens after
// five rounds while it was still reading.
it("gives the person's own plan more than the operator's", () => {
expect(TURN_LIMITS.user).toEqual({
tokens: 120_000,
wallMs: 150_000,
maxRounds: 12,
tokens: 165_000,
wallMs: 200_000,
maxRounds: 16,
});
expect(TURN_LIMITS.operator).toEqual({
tokens: 40_000,
wallMs: 90_000,
maxRounds: 6,
tokens: 55_000,
wallMs: 120_000,
maxRounds: 8,
});
});
it("keeps the final answer at the same share of each payer's tokens", () => {
expect(FINAL_ANSWER_TOKENS / TURN_LIMITS.user.tokens).toBeCloseTo(0.01);
expect(FINAL_ANSWER_TOKENS / TURN_LIMITS.operator.tokens).toBeCloseTo(0.03);
});
it("holds the turn's slot for at least the longest turn", async () => {
// The chat route's concurrency slot frees itself after its window; a
// window shorter than the wall time would let a third turn start while
// a long one is still running.
const { readFileSync } = await import("node:fs");
const route = readFileSync(
`${process.cwd()}/src/app/api/insights/chat/route.ts`,
"utf8",
);
expect(route).toMatch(
/COACH_TURN_SLOT_MS = TURN_LIMITS\.user\.wallMs \+ \d/,
);
});
});
describe("token accounting", () => {
@@ -73,9 +95,9 @@ describe("check()", () => {
effort: "medium",
initialInputTokens: 10_000,
});
// 34k spent; the next round with medium thinking and the final answer
// need about 8k more, past the operator's 40k.
budget.endRound({ tokens: 34_000, cachedTokens: 0, durationMs: 1_000 });
// 49k spent; the next round with medium thinking and the final answer
// need about 8k more, past the operator's 55k.
budget.endRound({ tokens: 49_000, cachedTokens: 0, durationMs: 1_000 });
expect(budget.check()).toBe("budget");
});
@@ -103,9 +125,9 @@ describe("check()", () => {
initialInputTokens: 1_000,
now: c.now,
});
c.advance(30_000);
c.advance(60_000);
budget.endRound({ tokens: 1_000, cachedTokens: 0, durationMs: 30_000 });
// 30 s elapsed + two typical 30 s rounds = 90 s: not over yet.
// 60 s elapsed + two typical 30 s rounds = 120 s: not over yet.
expect(budget.check()).toBeNull();
c.advance(1);
budget.endRound({ tokens: 1_000, cachedTokens: 900, durationMs: 30_000 });
+5
View File
@@ -30,6 +30,7 @@
* A clarifying question ends the turn without a prose round: the question
* is the reply.
*/
import { coachPromptCacheKey } from "@/lib/ai/coach/prompt-cache-key";
import type { CoachHistoryReach } from "@/lib/ai/coach/history-reach";
import { annotate } from "@/lib/logging/context";
import type { Locale } from "@/lib/i18n/config";
@@ -277,6 +278,9 @@ export async function runCoachToolLoop(args: {
),
});
const progress = createProgressTracker();
// v1.41.2 — one key for every round, so round two finds round one's prefix
// in the provider's prompt cache.
const cacheKey = coachPromptCacheKey(turn?.conversationId);
const messages: AiMessage[] = [...args.messages];
let system = args.system;
@@ -351,6 +355,7 @@ export async function runCoachToolLoop(args: {
: {}),
tools,
toolChoice: isFinal ? ("none" as const) : ("auto" as const),
cacheKey,
},
});
const result = fallback.result;
+13 -4
View File
@@ -46,17 +46,26 @@ export interface TurnLimits {
* Fixed per payer, not configurable: the person's own plan, key or local
* model gets room for a long "why" chain; a turn the operator pays for gets
* a third of the tokens and half the rounds.
*
* v1.41.2 — about a third more on both sides (was 120k / 150 s / 12 rounds
* and 40k / 90 s / 6 rounds). A real turn on a person's own plan stopped for
* tokens after five rounds while it was still reading, and the operator
* side moves with it so the split stays the same.
*/
export const TURN_LIMITS: Readonly<Record<TurnPayer, TurnLimits>> = {
user: { tokens: 120_000, wallMs: 150_000, maxRounds: 12 },
operator: { tokens: 40_000, wallMs: 90_000, maxRounds: 6 },
user: { tokens: 165_000, wallMs: 200_000, maxRounds: 16 },
operator: { tokens: 55_000, wallMs: 120_000, maxRounds: 8 },
};
/** The answer budget of one round, before any thinking budget. */
export const ROUND_ANSWER_TOKENS = 600;
/** What the final answer is given on top of its input. */
export const FINAL_ANSWER_TOKENS = 1_200;
/**
* What the final answer is given on top of its input. v1.41.2 — raised with
* the limits (was 1,200) so it keeps the same share of each payer's tokens:
* 1 % of the person's own budget, 3 % of the operator's.
*/
export const FINAL_ANSWER_TOKENS = 1_650;
/** Cached input counts at this share of its size. */
export const CACHED_INPUT_WEIGHT = 0.1;
@@ -462,7 +462,7 @@ describe("toStep — mapping", () => {
parsedArgs: parsed,
locale: "en",
})?.id,
).toBe("s48");
).toBe("s64");
expect(
toStep({
call: { id: "u", name: "get_everything", arguments: "{}" },
+2
View File
@@ -4,6 +4,7 @@
* completion over the full snapshot. A provider failure is classified into
* the structured `coach.*` code the stream answers with.
*/
import { coachPromptCacheKey } from "@/lib/ai/coach/prompt-cache-key";
import { annotate } from "@/lib/logging/context";
import type { Locale } from "@/lib/i18n/config";
import { localeLanguageNames as LANGUAGE_NAMES } from "@/lib/i18n/config";
@@ -652,6 +653,7 @@ export async function runTurnModel(args: {
// v1.22 (#89) — per-idle-gap timeout for the streaming local call.
timeoutMs: ctx.aiResponseTimeoutMs,
}),
cacheKey: coachPromptCacheKey(conversationId),
...(args.reasoning
? {
reasoning: args.reasoning,
+3 -2
View File
@@ -40,9 +40,10 @@ import { COMPARE_SERIES_TOOL_NAME } from "@/lib/ai/coach/tools/compare-series";
/**
* At most this many steps per turn; later calls run but show no step.
* v1.41 — 48: a budgeted turn runs up to twelve rounds.
* v1.41 — four per round of the longest turn: 48 for twelve rounds, 64 since
* v1.41.2 raised the cap to sixteen.
*/
export const MAX_TURN_STEPS = 48;
export const MAX_TURN_STEPS = 64;
const SCOPE_SOURCES: ReadonlySet<string> = new Set(
coachScopeSourceSchema.options,
+1 -1
View File
@@ -498,7 +498,7 @@ export type CoachStepReason =
| "invalid_arguments";
export interface CoachStep {
/** `s1`..`s48`, unique within a turn; frames upsert by id. */
/** `s1`..`s64`, unique within a turn; frames upsert by id. */
id: string;
tool: CoachToolName | "show_result" | "snapshot";
/** Closed catalog key, e.g. `coach.step.read`. */
+26 -3
View File
@@ -1,4 +1,4 @@
import { randomUUID } from "node:crypto";
import { createHash, randomUUID } from "node:crypto";
import { callTimeoutMs } from "./effective-timeout";
import { safeFetch } from "@/lib/safe-fetch";
import type {
@@ -431,6 +431,21 @@ export interface CodexAttemptDiagnostics {
workingSlug: string | null;
}
/**
* A UUID-shaped id derived from a cache key, for the session headers: the
* same key gives the same id, and the key itself never rides a header.
*/
function uuidFromKey(key: string): string {
const hex = createHash("sha256").update(`codex-session:${key}`).digest("hex");
return [
hex.slice(0, 8),
hex.slice(8, 12),
`4${hex.slice(13, 16)}`,
`${((parseInt(hex[16], 16) & 0x3) | 0x8).toString(16)}${hex.slice(17, 20)}`,
hex.slice(20, 32),
].join("-");
}
export class CodexClient implements AIProvider {
readonly type = "codex" as const;
/** See `AIProvider.responseTimeoutSeconds`; stamped by the resolver. */
@@ -650,8 +665,15 @@ export class CodexClient implements AIProvider {
slug: string,
wire: CodexReasoningWire,
): Promise<Response> {
const sessionId = randomUUID();
const threadId = randomUUID();
// v1.41.2 — calls that share a cache key share one session and thread
// id and send the key as `prompt_cache_key`, the way the official client
// sends its thread id (docs/codex-protocol-spec.md §3b/§3c). Before, every
// round of a Coach turn arrived as a new session with no key, so the
// backend had no reason to route it to the cache the previous round
// filled, and each round was charged its whole prefix again.
const cacheKey = params.cacheKey?.trim() || null;
const sessionId = cacheKey ? uuidFromKey(cacheKey) : randomUUID();
const threadId = cacheKey ? sessionId : randomUUID();
// v1.18.11 — fold vision inputs (Lab-OCR) into the user turn when present.
// The Codex/ChatGPT-OAuth backend accepts the `input_image` content block
@@ -713,6 +735,7 @@ export class CodexClient implements AIProvider {
: null,
store: false,
stream: true,
...(cacheKey ? { prompt_cache_key: cacheKey } : {}),
// Required field; empty array when not asking for reasoning. With
// reasoning on, the encrypted items come back so the next tool
// round can hand them over statelessly (`store` must stay false).
+17 -1
View File
@@ -11,6 +11,7 @@ import type {
import {
buildOpenAIMessages,
mapFinishReason,
parseCachedTokens,
raiseEmbeddedError,
readEmbeddedError,
sanitiseBodyExcerpt,
@@ -411,6 +412,7 @@ export class LocalOpenAICompatibleClient implements AIProvider {
}>;
usage?: {
total_tokens?: number;
prompt_tokens_details?: { cached_tokens?: number };
completion_tokens_details?: { reasoning_tokens?: number };
};
};
@@ -459,6 +461,11 @@ export class LocalOpenAICompatibleClient implements AIProvider {
return {
content,
tokensUsed: json.usage?.total_tokens ?? null,
// v1.41.2 — llama.cpp, vLLM and the hosted gateways this client also
// serves report a prompt-cache hit the Chat Completions way; without it
// the Coach turn budget counted a cached prefix in full. Ollama sends
// none, which leaves it null.
cachedInputTokens: parseCachedTokens(json),
model: this.config.model,
providerType: "local",
// v1.28.28 (#470) — surface why the model stopped so the JSON callers
@@ -543,7 +550,10 @@ export class LocalOpenAICompatibleClient implements AIProvider {
message?: { content?: string };
finish_reason?: string;
}>;
usage?: { total_tokens?: number };
usage?: {
total_tokens?: number;
prompt_tokens_details?: { cached_tokens?: number };
};
} | null;
// The server ignored `stream` and answered buffered — the embedded
// failure can ride this body exactly as it rides the non-streaming one.
@@ -577,6 +587,7 @@ export class LocalOpenAICompatibleClient implements AIProvider {
return {
content: buffered,
tokensUsed: json?.usage?.total_tokens ?? null,
cachedInputTokens: json ? parseCachedTokens(json) : null,
model: this.config.model,
providerType: "local",
finishReason: mapFinishReason(json?.choices?.[0]?.finish_reason),
@@ -589,6 +600,7 @@ export class LocalOpenAICompatibleClient implements AIProvider {
let sseBuffer = "";
let content = "";
let tokensUsed: number | null = null;
let cachedInputTokens: number | null = null;
let reasoningTokens: number | null = null;
let finishReason: string | undefined;
@@ -679,6 +691,7 @@ export class LocalOpenAICompatibleClient implements AIProvider {
}>;
usage?: {
total_tokens?: number;
prompt_tokens_details?: { cached_tokens?: number };
completion_tokens_details?: { reasoning_tokens?: number };
};
};
@@ -702,6 +715,8 @@ export class LocalOpenAICompatibleClient implements AIProvider {
if (typeof chunk.usage?.total_tokens === "number") {
tokensUsed = chunk.usage.total_tokens;
}
const cached = chunk.usage?.prompt_tokens_details?.cached_tokens;
if (typeof cached === "number") cachedInputTokens = cached;
const rt = chunk.usage?.completion_tokens_details?.reasoning_tokens;
if (typeof rt === "number") reasoningTokens = rt;
const chunkFinish = chunk.choices?.[0]?.finish_reason;
@@ -761,6 +776,7 @@ export class LocalOpenAICompatibleClient implements AIProvider {
return {
content,
tokensUsed,
cachedInputTokens,
model: this.config.model,
providerType: "local",
finishReason: mapFinishReason(finishReason),
+1 -1
View File
@@ -34,7 +34,7 @@ function isModernCanonicalModel(model: string): boolean {
);
}
function isCanonicalOpenAIEndpoint(baseUrl: string): boolean {
export function isCanonicalOpenAIEndpoint(baseUrl: string): boolean {
try {
const url = new URL(baseUrl);
return (
+8
View File
@@ -3,6 +3,7 @@ import { callTimeoutMs } from "./effective-timeout";
import { annotate } from "@/lib/logging/context";
import type { AIProvider, CompletionParams, CompletionResult } from "./types";
import {
isCanonicalOpenAIEndpoint,
selectOpenAIChatCompletionsCapabilities,
type OpenAIChatCompletionsCapabilities,
} from "./openai-capabilities";
@@ -208,6 +209,13 @@ export class OpenAIClient implements AIProvider {
: {}),
...(tools ? { tools } : {}),
...(params.toolChoice ? { tool_choice: params.toolChoice } : {}),
// v1.41.2 — the prompt-cache routing key `api.openai.com` documents
// (https://platform.openai.com/docs/guides/prompt-caching). Only
// there: a gateway or proxy behind another base URL may refuse a
// field it does not know.
...(params.cacheKey && isCanonicalOpenAIEndpoint(this.config.baseUrl)
? { prompt_cache_key: params.cacheKey }
: {}),
// #1126 — the gateway entry's reasoning setting; Default sends
// nothing. `api.openai.com` and Codex never get it. v1.41 — a call
// that carries its own resolved level (`params.reasoning`) wins.
+3 -1
View File
@@ -150,7 +150,9 @@ export function parseOpenAIToolCalls(
}));
}
export function parseCachedTokens(json: OpenAIResponseJson): number | null {
export function parseCachedTokens(json: {
usage?: { prompt_tokens_details?: { cached_tokens?: number } };
}): number | null {
return json.usage?.prompt_tokens_details?.cached_tokens ?? null;
}
+16
View File
@@ -374,6 +374,16 @@ export interface CompletionParams {
* client sends exactly what it sent before the field existed.
*/
reasoning?: { effort: ReasoningLevel; summaries: boolean };
/**
* v1.41.2 — a stable key for calls that resend the same prompt prefix (the
* rounds of one Coach turn, the turns of one conversation). Providers that
* route a request to a prompt cache by key read it: Codex as
* `prompt_cache_key` and its session headers, `api.openai.com` as
* `prompt_cache_key`. Without it a Codex call is routed afresh every time
* and a later round pays for its whole prefix again. An opaque value: the
* caller hashes whatever it derives the key from. Absent → nothing is sent.
*/
cacheKey?: string;
/**
* v1.41 — live reasoning events, called as they arrive on clients that read
* the provider's stream, or once after the reply on clients that do not.
@@ -390,6 +400,12 @@ export interface CompletionResult {
* from the provider's cache, where the provider reports it (Anthropic
* `cache_read_input_tokens`, OpenAI/Codex `cached_tokens`). Null / absent
* when the provider does not report it or the prefix did not hit.
*
* v1.41.2 — always a share of `tokensUsed`, never beside it: the turn
* budget and the daily ledger charge `tokensUsed - cachedInputTokens` at
* full weight and the cached share at a tenth. A client whose provider
* reports cached input outside its input count (Anthropic) adds it to
* `tokensUsed`.
*/
cachedInputTokens?: number | null;
model: string;