From 8d23c79ef504992a20e8e066e2289fa7da6a85ae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Tue, 6 Oct 2026 19:26:28 +0200 Subject: [PATCH 1/7] feat(ai): add the reasoning support matrix and a per-endpoint dialect cache reasoningSupport() reads what each provider can do with reasoning from the provider type and model name: effort, summaries, live summaries, state that must travel between tool rounds, and whether off is a real off. Unknown names are treated optimistically. The dialect cache records what an endpoint refused (keyed by provider, endpoint and model) so a client retries with the lesser request once and never pays the refused round-trip again. Downgrades are marked on the wide event with meta keys only. --- .../reasoning/__tests__/dialect-cache.test.ts | 52 +++ .../ai/reasoning/__tests__/support.test.ts | 171 ++++++++++ src/lib/ai/reasoning/dialect-cache.ts | 77 +++++ src/lib/ai/reasoning/support.ts | 297 ++++++++++++++++++ 4 files changed, 597 insertions(+) create mode 100644 src/lib/ai/reasoning/__tests__/dialect-cache.test.ts create mode 100644 src/lib/ai/reasoning/__tests__/support.test.ts create mode 100644 src/lib/ai/reasoning/dialect-cache.ts create mode 100644 src/lib/ai/reasoning/support.ts diff --git a/src/lib/ai/reasoning/__tests__/dialect-cache.test.ts b/src/lib/ai/reasoning/__tests__/dialect-cache.test.ts new file mode 100644 index 000000000..857a16cbf --- /dev/null +++ b/src/lib/ai/reasoning/__tests__/dialect-cache.test.ts @@ -0,0 +1,52 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; + +const annotations = vi.hoisted( + () => [] as Array<{ action?: unknown; meta?: Record }>, +); +vi.mock("@/lib/logging/context", () => ({ + annotate: (fields: { action?: unknown; meta?: Record }) => + annotations.push(fields), +})); + +import { + annotateReasoningDowngrade, + learnedReasoningDialect, + rememberReasoningDialect, + resetReasoningDialectCache, +} from "../dialect-cache"; + +beforeEach(() => { + resetReasoningDialectCache(); + annotations.length = 0; +}); + +describe("reasoning dialect cache", () => { + it("keys on provider, endpoint and model together", () => { + const key = { provider: "local", endpoint: "http://a/v1", model: "m1" }; + rememberReasoningDialect(key, "none"); + expect(learnedReasoningDialect(key)).toBe("none"); + expect(learnedReasoningDialect({ ...key, model: "m2" })).toBeUndefined(); + expect( + learnedReasoningDialect({ ...key, endpoint: "http://b/v1" }), + ).toBeUndefined(); + expect( + learnedReasoningDialect({ ...key, provider: "openai" }), + ).toBeUndefined(); + resetReasoningDialectCache(); + expect(learnedReasoningDialect(key)).toBeUndefined(); + }); + + it("marks a downgrade with meta keys only, never the action", () => { + annotateReasoningDowngrade("codex", "high", "medium"); + expect(annotations).toEqual([ + { + meta: { + ai_reasoning_downgraded: true, + ai_reasoning_downgraded_provider: "codex", + ai_reasoning_downgraded_from: "high", + ai_reasoning_downgraded_to: "medium", + }, + }, + ]); + }); +}); diff --git a/src/lib/ai/reasoning/__tests__/support.test.ts b/src/lib/ai/reasoning/__tests__/support.test.ts new file mode 100644 index 000000000..3f2fc7dcb --- /dev/null +++ b/src/lib/ai/reasoning/__tests__/support.test.ts @@ -0,0 +1,171 @@ +import { describe, expect, it } from "vitest"; +import { + anthropicModelVersion, + anthropicRefusesSampling, + anthropicThinkingDialect, + anthropicThinksAlways, + isOpenRouterEndpoint, + levelOfWireEffort, + openAIOffEffort, + openAIReasoningFamily, + reasoningSupport, + reasoningTitleOf, + splitThinkTags, +} from "../support"; + +describe("anthropicModelVersion", () => { + it.each([ + ["claude-sonnet-4-6", { major: 4, minor: 6 }], + ["claude-opus-4-7", { major: 4, minor: 7 }], + ["claude-opus-5", { major: 5, minor: 0 }], + ["claude-opus-5-5", { major: 5, minor: 5 }], + ["claude-fable-5-1", { major: 5, minor: 1 }], + ["claude-haiku-4-5", { major: 4, minor: 5 }], + // An eight-digit segment is a snapshot date, not a minor. + ["claude-opus-4-20250514", { major: 4, minor: 0 }], + ["claude-opus-4-1-20250805", { major: 4, minor: 1 }], + ["claude-sonnet-4.5", { major: 4, minor: 5 }], + ["anthropic/claude-sonnet-4.6", { major: 4, minor: 6 }], + ["claude-3-7-sonnet-latest", { major: 3, minor: 7 }], + ["claude-3-5-sonnet-latest", { major: 3, minor: 5 }], + ])("%s", (model, expected) => { + expect(anthropicModelVersion(model)).toEqual(expected); + }); + + it("returns null for a name it cannot read", () => { + expect(anthropicModelVersion("my-claude-proxy")).toBeNull(); + }); +}); + +describe("anthropicThinkingDialect", () => { + it.each([ + ["claude-3-5-sonnet-latest", "none"], + ["claude-3-7-sonnet-latest", "manual"], + ["claude-opus-4-20250514", "manual"], + ["claude-haiku-4-5", "manual"], + ["claude-sonnet-4-5", "manual"], + // budget_tokens is deprecated on 4.6 and refused from 4.7 on. + ["claude-sonnet-4-6", "adaptive"], + ["claude-opus-4-7", "adaptive"], + ["claude-opus-5", "adaptive"], + ["claude-sonnet-5", "adaptive"], + // Unknown names try the current form first and learn from a refusal. + ["my-claude-proxy", "adaptive"], + ])("%s → %s", (model, dialect) => { + expect(anthropicThinkingDialect(model)).toBe(dialect); + }); + + it("refuses sampling from 4.7 on, never for an unknown name", () => { + expect(anthropicRefusesSampling("claude-sonnet-4-6")).toBe(false); + expect(anthropicRefusesSampling("claude-opus-4-7")).toBe(true); + expect(anthropicRefusesSampling("claude-sonnet-5")).toBe(true); + expect(anthropicRefusesSampling("claude-opus-5-5")).toBe(true); + expect(anthropicRefusesSampling("my-claude-proxy")).toBe(false); + }); + + it("thinks always from the 5 generation on", () => { + expect(anthropicThinksAlways("claude-opus-4-8")).toBe(false); + expect(anthropicThinksAlways("claude-opus-5")).toBe(true); + expect(anthropicThinksAlways("claude-sonnet-5")).toBe(true); + }); +}); + +describe("OpenAI families", () => { + it.each([ + ["gpt-5.5", "gpt-5.1+", "none"], + ["gpt-5.1-mini", "gpt-5.1+", "none"], + ["gpt-5", "gpt-5", "minimal"], + ["gpt-5-mini", "gpt-5", "minimal"], + ["o3", "o", "low"], + ["o4-mini", "o", "low"], + ] as const)("%s", (model, family, off) => { + expect(openAIReasoningFamily(model)).toBe(family); + expect(openAIOffEffort(family)).toBe(off); + }); + + it("does not treat a non-reasoning model as one", () => { + expect(openAIReasoningFamily("gpt-4o")).toBeNull(); + expect(openAIReasoningFamily("gpt-4.1-mini")).toBeNull(); + }); + + it("recognises OpenRouter by host only", () => { + expect(isOpenRouterEndpoint("https://openrouter.ai/api/v1")).toBe(true); + expect(isOpenRouterEndpoint("https://eu.openrouter.ai/api/v1")).toBe(true); + expect(isOpenRouterEndpoint("https://openrouter.ai.evil.example/v1")).toBe( + false, + ); + expect(isOpenRouterEndpoint("not a url")).toBe(false); + }); +}); + +describe("reasoningSupport", () => { + it("reports the matrix per provider", () => { + expect(reasoningSupport("codex", "gpt-5.5")).toEqual({ + effort: true, + summaries: true, + liveSummaries: true, + stateRoundTrip: true, + offIsReal: true, + }); + expect(reasoningSupport("anthropic", "claude-opus-5").offIsReal).toBe( + false, + ); + expect(reasoningSupport("anthropic", "claude-sonnet-4-6").offIsReal).toBe( + true, + ); + expect( + reasoningSupport("anthropic", "claude-3-5-sonnet-latest").effort, + ).toBe(false); + expect(reasoningSupport("admin-key", "gpt-4o").effort).toBe(false); + expect(reasoningSupport("admin-key", "gpt-5").offIsReal).toBe(false); + expect( + reasoningSupport("openai-compatible", "x", { + baseUrl: "https://openrouter.ai/api/v1", + }).stateRoundTrip, + ).toBe(true); + expect( + reasoningSupport("openai-compatible", "x", { + baseUrl: "https://litellm.example.com/v1", + }).stateRoundTrip, + ).toBe(false); + expect(reasoningSupport("local", "qwen3").liveSummaries).toBe(true); + expect(reasoningSupport("none", "").effort).toBe(false); + }); +}); + +describe("parsing helpers", () => { + it("reads a bold first line as the title, and nothing else", () => { + expect(reasoningTitleOf("**Checking the window**\n\nBody")).toBe( + "Checking the window", + ); + expect(reasoningTitleOf("**Checking the window**")).toBe( + "Checking the window", + ); + expect(reasoningTitleOf("**Checking the")).toBeNull(); + expect(reasoningTitleOf("Plain first line\n**Later**")).toBeNull(); + }); + + it("splits a leading think block and leaves later mentions alone", () => { + expect(splitThinkTags("a plan\n\nThe answer.")).toEqual({ + content: "The answer.", + reasoning: "a plan", + }); + expect(splitThinkTags("The answer mentions .")).toEqual({ + content: "The answer mentions .", + reasoning: null, + }); + // Unclosed: the budget ran out inside the reasoning. + expect(splitThinkTags("still going")).toEqual({ + content: "", + reasoning: "still going", + }); + }); + + it("maps wire efforts back onto levels", () => { + expect(levelOfWireEffort("none")).toBe("off"); + expect(levelOfWireEffort("minimal")).toBe("off"); + expect(levelOfWireEffort("medium")).toBe("medium"); + expect(levelOfWireEffort("xhigh")).toBe("high"); + expect(levelOfWireEffort(null)).toBe("off"); + }); +}); diff --git a/src/lib/ai/reasoning/dialect-cache.ts b/src/lib/ai/reasoning/dialect-cache.ts new file mode 100644 index 000000000..c9024da09 --- /dev/null +++ b/src/lib/ai/reasoning/dialect-cache.ts @@ -0,0 +1,77 @@ +/** + * v1.41 — what an endpoint answered when it was asked to reason, learned at + * runtime and kept for the life of the process. + * + * Every client that sends reasoning parameters to a wire it does not control + * (a gateway, a local server, a model name the client has never seen, a Codex + * slug whose accepted efforts moved) asks this cache first. When the endpoint + * rejects a parameter the client retries once with the lesser request and + * records the lesser request here, so the next call does not pay the rejected + * round-trip again. A successful call never has to be recorded: the default + * is "send what was asked". + * + * Keyed by provider, endpoint and model, because the same gateway can serve a + * model that reasons and one that does not, and the same model name can sit + * behind two endpoints with different parameter rules. The values are short + * strings each client defines for itself; nothing outside the client reads + * them. + * + * Pattern borrowed from `../json-dialect.ts`, which learns the JSON-mode flag + * the same way. Server-only: the provider clients are its only importers. + */ +import { annotate } from "@/lib/logging/context"; + +export interface ReasoningDialectKey { + /** The client that learned it (`codex`, `anthropic`, `openai`, `local`). */ + provider: string; + /** Base URL or endpoint constant. */ + endpoint: string; + model: string; +} + +const learned = new Map(); + +function cacheKey(key: ReasoningDialectKey): string { + return `${key.provider}\u0000${key.endpoint}\u0000${key.model}`; +} + +/** The learned dialect value for `key`, or `undefined` when nothing was learned. */ +export function learnedReasoningDialect( + key: ReasoningDialectKey, +): string | undefined { + return learned.get(cacheKey(key)); +} + +/** Record what `key` accepts after it refused a richer request. */ +export function rememberReasoningDialect( + key: ReasoningDialectKey, + value: string, +): void { + learned.set(cacheKey(key), value); +} + +/** Test hook: forget everything learned. */ +export function resetReasoningDialectCache(): void { + learned.clear(); +} + +/** + * Mark the request's wide event with a reasoning downgrade. Meta keys only, + * never an action: the route or job owns the action name dashboards pin on. + * The values are our own vocabulary (provider tag, wire effort), never model + * or upstream text. + */ +export function annotateReasoningDowngrade( + provider: string, + from: string, + to: string, +): void { + annotate({ + meta: { + ai_reasoning_downgraded: true, + ai_reasoning_downgraded_provider: provider, + ai_reasoning_downgraded_from: from, + ai_reasoning_downgraded_to: to, + }, + }); +} diff --git a/src/lib/ai/reasoning/support.ts b/src/lib/ai/reasoning/support.ts new file mode 100644 index 000000000..b1d6d0615 --- /dev/null +++ b/src/lib/ai/reasoning/support.ts @@ -0,0 +1,297 @@ +/** + * v1.41 — what each provider can do with reasoning, read from the provider + * type and the model name alone. + * + * No admin keeps a model list. A name this file does not recognise is treated + * optimistically: the client sends the request, and if the endpoint refuses a + * parameter it retries without it and learns the refusal per endpoint and + * model (`dialect-cache.ts`). What is reported here is therefore the starting + * assumption, not a promise; the settings use `offIsReal` to label the lowest + * option "Off" or "Minimal". + * + * The wire facts behind each branch, with sources: + * + * - Codex backend: Responses API `reasoning: { effort, summary }` plus + * `include: ["reasoning.encrypted_content"]` for the stateless round trip. + * A live probe on 2026-10-06 against the current slug (`gpt-5.5`) answered + * `effort: "none"` with a non-reasoning reply and rejected `minimal` with + * "Supported values are: 'none', 'low', 'medium', 'high', and 'xhigh'", + * so "off" is real there. https://developers.openai.com/api/docs/guides/reasoning, + * https://developers.openai.com/cookbook/examples/responses_api/reasoning_items + * - OpenAI Chat Completions: `reasoning_effort`, no summaries and no + * reasoning items to hand back between tool rounds, so reasoning is only + * sent on rounds without tools. `none` exists from gpt-5.1 on; gpt-5 knows + * `minimal`; the o-series starts at `low`. + * https://platform.openai.com/docs/api-reference/chat/create + * - Anthropic: manual `thinking: { type: "enabled", budget_tokens }` up to + * the 4.5 generation; adaptive `thinking: { type: "adaptive" }` with + * `output_config.effort` from 4.6 on (`budget_tokens` is deprecated on 4.6 + * and refused from 4.7). From the 5 generation thinking is on whatever the + * request says, so "off" becomes the lowest effort. Sampling parameters + * are refused from 4.7 on. + * https://platform.claude.com/docs/en/build-with-claude/extended-thinking, + * https://platform.claude.com/docs/en/build-with-claude/adaptive-thinking, + * https://platform.claude.com/docs/en/build-with-claude/effort + * - OpenRouter: `reasoning: { effort }` with `none` and `minimal` accepted, + * reply `reasoning` (text) and `reasoning_details[]`, which must go back + * unmodified on the assistant message of a tool round. + * https://openrouter.ai/docs/use-cases/reasoning-tokens + * - Other OpenAI-compatible gateways and local servers: `reasoning_effort`, + * reply `reasoning_content` (llama.cpp, LiteLLM) or `reasoning` (vLLM, + * Ollama, LM Studio), some models inline `…`. + * https://docs.litellm.ai/docs/reasoning_content, + * https://docs.vllm.ai/en/latest/features/reasoning_outputs.html, + * https://docs.ollama.com/api/openai-compatibility, + * https://github.com/ggml-org/llama.cpp/blob/master/tools/server/README.md + * + * Client-safe: no server import. + */ +import type { ProviderType } from "../types"; +import type { ReasoningLevel } from "./levels"; + +export interface ReasoningSupport { + /** The provider takes an effort (or a budget) at all. */ + effort: boolean; + /** It returns readable summaries of the reasoning. */ + summaries: boolean; + /** Summaries arrive while the call runs, not only after it. */ + liveSummaries: boolean; + /** Reasoning state has to travel between tool rounds (`providerState`). */ + stateRoundTrip: boolean; + /** "Off" switches reasoning off; false means it falls to the lowest level. */ + offIsReal: boolean; +} + +const NONE: ReasoningSupport = { + effort: false, + summaries: false, + liveSummaries: false, + stateRoundTrip: false, + offIsReal: true, +}; + +// ── Anthropic ─────────────────────────────────────────────────────────────── + +export interface AnthropicModelVersion { + major: number; + minor: number; +} + +/** + * The generation of a Claude model id: `claude--[-][-]` + * from Claude 4 on (`claude-sonnet-4-6`, `claude-opus-5`, `claude-opus-4-20250514`), + * `claude-[-]-` before (`claude-3-7-sonnet-latest`). A + * dotted minor (`claude-sonnet-4.5`, the gateway spelling) reads the same. An + * eight-digit segment is a snapshot date, not a minor. `null` for anything + * else (a proxy alias, a typo): the client then tries the current dialect and + * learns from the answer. + */ +export function anthropicModelVersion( + model: string, +): AnthropicModelVersion | null { + const m = model.toLowerCase().replace(/^anthropic\//, ""); + const modern = + /^claude-(?:opus|sonnet|haiku|fable|mythos)-(\d{1,2})(?:[-.](\d{1,2}))?(?:$|[-@:])/.exec( + m, + ); + if (modern) { + return { major: Number(modern[1]), minor: Number(modern[2] ?? 0) }; + } + const legacy = /^claude-(\d)(?:[-.](\d))?-(?:opus|sonnet|haiku)/.exec(m); + if (legacy) { + return { major: Number(legacy[1]), minor: Number(legacy[2] ?? 0) }; + } + return null; +} + +function atLeast( + v: AnthropicModelVersion, + major: number, + minor: number, +): boolean { + return v.major > major || (v.major === major && v.minor >= minor); +} + +/** + * How a Claude model is asked to think. `manual` is the budget form (up to + * 4.5, and 3.7), `adaptive` the effort form (4.6 on, and every name this + * function cannot read), `none` a model that cannot think (3.5 and older). + */ +export type AnthropicThinkingDialect = "adaptive" | "manual" | "none"; + +export function anthropicThinkingDialect( + model: string, +): AnthropicThinkingDialect { + const v = anthropicModelVersion(model); + if (!v) return "adaptive"; + if (v.major < 3) return "none"; + if (v.major === 3) return v.minor >= 7 ? "manual" : "none"; + return atLeast(v, 4, 6) ? "adaptive" : "manual"; +} + +/** + * True when the model refuses `temperature` / `top_p` / `top_k` (4.7 on; the + * 5 generation refuses any value, including the default). Unknown names + * return false: the client keeps today's wire and learns from a refusal. + */ +export function anthropicRefusesSampling(model: string): boolean { + const v = anthropicModelVersion(model); + return v !== null && atLeast(v, 4, 7); +} + +/** True from the 5 generation on: thinking runs whatever the request says. */ +export function anthropicThinksAlways(model: string): boolean { + const v = anthropicModelVersion(model); + return v !== null && v.major >= 5; +} + +// ── OpenAI ────────────────────────────────────────────────────────────────── + +/** + * The reasoning family of an OpenAI model id, or `null` for a model that does + * not reason (`gpt-4o`, `gpt-4.1`). `gpt-5.1+` knows `none`, bare `gpt-5` + * (and its mini / nano) stops at `minimal`, the o-series at `low`. + */ +export type OpenAIReasoningFamily = "gpt-5.1+" | "gpt-5" | "o" | null; + +export function openAIReasoningFamily(model: string): OpenAIReasoningFamily { + const m = model.toLowerCase().replace(/^openai\//, ""); + const gpt5 = /^gpt-5(?:\.(\d+))?(?:$|-)/.exec(m); + if (gpt5) return gpt5[1] && Number(gpt5[1]) >= 1 ? "gpt-5.1+" : "gpt-5"; + if (/^o[134](?:$|-)/.test(m)) return "o"; + return null; +} + +/** The lowest `reasoning_effort` an OpenAI reasoning family accepts. */ +export function openAIOffEffort( + family: Exclude, +): "none" | "minimal" | "low" { + return family === "gpt-5.1+" + ? "none" + : family === "gpt-5" + ? "minimal" + : "low"; +} + +/** True for an OpenRouter base URL, whose reasoning dialect differs. */ +export function isOpenRouterEndpoint(baseUrl: string): boolean { + try { + const host = new URL(baseUrl).hostname.toLowerCase(); + return host === "openrouter.ai" || host.endsWith(".openrouter.ai"); + } catch { + return false; + } +} + +// ── Support matrix ────────────────────────────────────────────────────────── + +/** + * What `providerType` serving `model` can do with reasoning. `baseUrl` only + * matters for the gateway tag (OpenRouter has its own dialect) and the + * operator's OpenAI provider (a non-OpenAI base URL is a gateway in all but + * name). + */ +export function reasoningSupport( + providerType: ProviderType, + model: string, + opts: { baseUrl?: string } = {}, +): ReasoningSupport { + switch (providerType) { + case "codex": + return { + effort: true, + summaries: true, + liveSummaries: true, + stateRoundTrip: true, + offIsReal: true, + }; + case "anthropic": { + const dialect = anthropicThinkingDialect(model); + if (dialect === "none") return NONE; + return { + effort: true, + summaries: true, + liveSummaries: false, + stateRoundTrip: true, + offIsReal: !anthropicThinksAlways(model), + }; + } + case "admin-key": { + const family = openAIReasoningFamily(model); + if (!family) return NONE; + return { + effort: true, + summaries: false, + liveSummaries: false, + stateRoundTrip: false, + offIsReal: family === "gpt-5.1+", + }; + } + case "openai-compatible": { + const openRouter = opts.baseUrl + ? isOpenRouterEndpoint(opts.baseUrl) + : false; + return { + effort: true, + summaries: true, + liveSummaries: false, + stateRoundTrip: openRouter, + offIsReal: true, + }; + } + case "local": + return { + effort: true, + summaries: true, + liveSummaries: true, + stateRoundTrip: false, + offIsReal: true, + }; + case "none": + return NONE; + } +} + +// ── Shared parsing helpers ────────────────────────────────────────────────── + +/** + * The title of a reasoning summary: its first line when that line is wholly + * bold (`**Checking the window**`), the convention the Responses API + * summaries follow. `null` when the summary has no such line yet or at all. + */ +export function reasoningTitleOf(text: string): string | null { + const m = /^\s*\*\*([^*\n]+?)\*\*\s*(?:\n|$)/.exec(text); + return m ? m[1].trim() : null; +} + +/** + * Split a reply that inlines its reasoning as a leading `…` + * block (some local models do). Only a block at the very start counts; a + * mention later in the prose stays prose. An unclosed block means the answer + * budget ran out inside the reasoning: everything is reasoning, no answer. + */ +export function splitThinkTags(content: string): { + content: string; + reasoning: string | null; +} { + const m = /^\s*([\s\S]*?)(<\/think>|$)/.exec(content); + if (!m) return { content, reasoning: null }; + const reasoning = m[1].trim(); + const rest = m[2] ? content.slice(m[0].length).replace(/^\s+/, "") : ""; + return { content: rest, reasoning: reasoning.length > 0 ? reasoning : null }; +} + +/** The level a provider's wire effort corresponds to, for `downgradedTo`. */ +export function levelOfWireEffort(effort: string | null): ReasoningLevel { + switch (effort) { + case "low": + case "medium": + case "high": + return effort; + case "xhigh": + case "max": + return "high"; + default: + return "off"; + } +} From d87b8204f365054e09fc08717c850cc303be8934 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Tue, 6 Oct 2026 19:26:28 +0200 Subject: [PATCH 2/7] feat(ai): send reasoning on the Codex wire and hand its items back between tool rounds A call that carries a reasoning level sends reasoning {effort, summary} with include [reasoning.encrypted_content]; off is effort none. Summary titles are reported as soon as the bold first line closes, the text when the part is done. Reasoning items come back as provider state and are re-inserted before the next round's message and calls, only for the slug that made them. A 400 that names a reasoning parameter is answered with the lesser request the backend lists (no summary, the nearest supported effort, no reasoning), remembered per slug. Calls without a level keep reasoning null and include []. --- src/lib/ai/__tests__/codex-reasoning.test.ts | 404 ++++++++++++++++++ .../fixtures/reasoning/codex-after-tool.sse | 51 +++ .../reasoning/codex-effort-rejected.json | 8 + .../reasoning/codex-summary-rejected.json | 8 + .../fixtures/reasoning/codex-summary.sse | 81 ++++ .../fixtures/reasoning/codex-tool-round.sse | 39 ++ src/lib/ai/codex-client.ts | 337 ++++++++++++++- 7 files changed, 906 insertions(+), 22 deletions(-) create mode 100644 src/lib/ai/__tests__/codex-reasoning.test.ts create mode 100644 src/lib/ai/__tests__/fixtures/reasoning/codex-after-tool.sse create mode 100644 src/lib/ai/__tests__/fixtures/reasoning/codex-effort-rejected.json create mode 100644 src/lib/ai/__tests__/fixtures/reasoning/codex-summary-rejected.json create mode 100644 src/lib/ai/__tests__/fixtures/reasoning/codex-summary.sse create mode 100644 src/lib/ai/__tests__/fixtures/reasoning/codex-tool-round.sse diff --git a/src/lib/ai/__tests__/codex-reasoning.test.ts b/src/lib/ai/__tests__/codex-reasoning.test.ts new file mode 100644 index 000000000..4b640e5f0 --- /dev/null +++ b/src/lib/ai/__tests__/codex-reasoning.test.ts @@ -0,0 +1,404 @@ +import { readFileSync } from "node:fs"; +import { join } from "node:path"; +import { beforeEach, describe, expect, it, vi } from "vitest"; +import { CodexClient } from "../codex-client"; +import { clearCodexSlugCache } from "../codex-slug-cache"; +import { resetReasoningDialectCache } from "../reasoning/dialect-cache"; +import type { + AiMessage, + AiReasoningEvent, + CompletionParams, + CompletionResult, +} from "../types"; + +/** + * v1.41 — Codex reasoning at the wire. The SSE fixtures were recorded against + * the live backend on 2026-10-06 with throwaway prompts (no health data) and + * sanitised: encrypted reasoning content replaced by placeholders, account + * identifiers and echoed instructions removed. The 400 bodies are the + * backend's own answers to an unsupported effort and summary value. + */ +const FIXTURES = join(__dirname, "fixtures/reasoning"); +const fixture = (name: string) => readFileSync(join(FIXTURES, name), "utf8"); + +function sseResponse(body: string): Response { + const encoder = new TextEncoder(); + // Split into uneven chunks so event boundaries fall mid-chunk, as on a + // real connection. + const chunks: string[] = []; + for (let i = 0; i < body.length; i += 173) + chunks.push(body.slice(i, i + 173)); + const stream = new ReadableStream({ + start(controller) { + for (const c of chunks) controller.enqueue(encoder.encode(c)); + controller.close(); + }, + }); + return new Response(stream, { + status: 200, + headers: { "content-type": "text/event-stream" }, + }); +} + +function errorResponse(status: number, body: string): Response { + return new Response(body, { status }); +} + +function client() { + return new CodexClient({ + accessToken: "test-token", + accountId: "acct-test", + onTokenRefresh: vi + .fn() + .mockResolvedValue({ accessToken: "x", accountId: "acct-test" }), + slugChain: ["gpt-5.5"], + }); +} + +function params(extra: Partial = {}): CompletionParams { + return { + system: "You are a terse assistant.", + messages: [{ role: "user", content: "How long was the train moving?" }], + ...extra, + }; +} + +function sentBody(mock: ReturnType, call = 0) { + return JSON.parse(mock.mock.calls[call][1].body); +} + +beforeEach(() => { + vi.restoreAllMocks(); + clearCodexSlugCache(); + resetReasoningDialectCache(); +}); + +describe("Codex reasoning request", () => { + it("sends today's wire when the call asks for no reasoning", async () => { + const mock = vi + .fn() + .mockImplementation(async () => + sseResponse(fixture("codex-summary.sse")), + ); + vi.stubGlobal("fetch", mock); + const onReasoning = vi.fn(); + const result = await client().generateCompletion(params({ onReasoning })); + const body = sentBody(mock); + expect(body.reasoning).toBeNull(); + expect(body.include).toEqual([]); + // No reasoning asked → no events, no reasoning block, no state. + expect(onReasoning).not.toHaveBeenCalled(); + expect(result.reasoning).toBeUndefined(); + expect(result.providerState).toBeUndefined(); + }); + + it("asks for effort, a summary and the encrypted items", async () => { + const mock = vi + .fn() + .mockImplementation(async () => + sseResponse(fixture("codex-summary.sse")), + ); + vi.stubGlobal("fetch", mock); + await client().generateCompletion( + params({ reasoning: { effort: "medium", summaries: true } }), + ); + const body = sentBody(mock); + expect(body.reasoning).toEqual({ effort: "medium", summary: "auto" }); + expect(body.include).toEqual(["reasoning.encrypted_content"]); + expect(body.store).toBe(false); + expect(body.stream).toBe(true); + }); + + it("leaves the summary out when none is wanted", async () => { + const mock = vi + .fn() + .mockImplementation(async () => + sseResponse(fixture("codex-summary.sse")), + ); + vi.stubGlobal("fetch", mock); + await client().generateCompletion( + params({ reasoning: { effort: "high", summaries: false } }), + ); + expect(sentBody(mock).reasoning).toEqual({ effort: "high" }); + }); + + it("switches reasoning off with effort none, without encrypted items", async () => { + const mock = vi + .fn() + .mockImplementation(async () => + sseResponse(fixture("codex-after-tool.sse")), + ); + vi.stubGlobal("fetch", mock); + const result = await client().generateCompletion( + params({ reasoning: { effort: "off", summaries: true } }), + ); + const body = sentBody(mock); + expect(body.reasoning).toEqual({ effort: "none" }); + expect(body.include).toEqual([]); + expect(result.reasoning?.downgradedTo).toBeUndefined(); + }); +}); + +describe("Codex reasoning stream", () => { + it("reports the title live, then the text, then done", async () => { + vi.stubGlobal( + "fetch", + vi + .fn() + .mockImplementation(async () => + sseResponse(fixture("codex-summary.sse")), + ), + ); + const events: AiReasoningEvent[] = []; + const result = await client().generateCompletion( + params({ + reasoning: { effort: "medium", summaries: true }, + onReasoning: (e) => events.push(e), + }), + ); + expect(events).toEqual([ + { kind: "title", text: "Calculating elapsed time minus offset" }, + { kind: "text", text: "**Calculating elapsed time minus offset**" }, + { kind: "done", text: "" }, + ]); + expect(result.content).toBe( + "The train was moving for 3 hours and 10 minutes.", + ); + expect(result.reasoning).toEqual({ + summary: ["**Calculating elapsed time minus offset**"], + tokens: 33, + }); + // Reasoning tokens are inside the billed total. + expect(result.tokensUsed).toBe(101); + }); + + it("keeps the reasoning items of a tool round as provider state", async () => { + vi.stubGlobal( + "fetch", + vi + .fn() + .mockImplementation(async () => + sseResponse(fixture("codex-tool-round.sse")), + ), + ); + const result = await client().generateCompletion( + params({ reasoning: { effort: "low", summaries: true } }), + ); + expect(result.finishReason).toBe("tool_calls"); + expect(result.toolCalls).toEqual([ + { + id: "call_edjM3le4GcXAsF7vywK4kQgY", + name: "get_weather", + arguments: '{"city":"Paris"}', + }, + ]); + expect(result.reasoning).toEqual({ summary: [], tokens: 16 }); + expect(result.providerState?.providerType).toBe("codex"); + expect(result.providerState?.model).toBe("gpt-5.5"); + expect(result.providerState?.items).toHaveLength(1); + expect(result.providerState?.items[0]).toMatchObject({ + type: "reasoning", + id: "rs_03743ca81429e023016ac52522f51c87d297ef373ae4b945c9", + encrypted_content: expect.stringMatching(/^enc-fixture-/), + }); + }); +}); + +describe("Codex reasoning round trip", () => { + async function firstRound(): Promise { + vi.stubGlobal( + "fetch", + vi + .fn() + .mockImplementation(async () => + sseResponse(fixture("codex-tool-round.sse")), + ), + ); + return client().generateCompletion( + params({ reasoning: { effort: "low", summaries: true } }), + ); + } + + function secondRoundMessages(first: CompletionResult): AiMessage[] { + return [ + { role: "user", content: "What is the weather in the capital?" }, + { + role: "assistant", + content: first.content, + toolCalls: first.toolCalls, + providerState: first.providerState, + }, + { + role: "tool", + toolCallId: first.toolCalls![0].id, + content: '{"tempC":18,"sky":"cloudy"}', + }, + ]; + } + + it("hands the reasoning item back unchanged before the call it led to", async () => { + const first = await firstRound(); + const mock = vi + .fn() + .mockImplementation(async () => + sseResponse(fixture("codex-after-tool.sse")), + ); + vi.stubGlobal("fetch", mock); + const second = await client().generateCompletion( + params({ + messages: secondRoundMessages(first), + reasoning: { effort: "low", summaries: true }, + }), + ); + const input = sentBody(mock).input as Array>; + expect(input.map((i) => i.type)).toEqual([ + "message", + "reasoning", + "message", + "function_call", + "function_call_output", + ]); + expect(input[1]).toEqual(first.providerState!.items[0]); + expect(second.content).toBe("Paris weather: 18°C, cloudy."); + }); + + it("drops state made for another slug or by another provider", async () => { + const first = await firstRound(); + const mock = vi + .fn() + .mockImplementation(async () => + sseResponse(fixture("codex-after-tool.sse")), + ); + vi.stubGlobal("fetch", mock); + const messages = secondRoundMessages(first); + messages[1] = { + ...messages[1], + providerState: { ...first.providerState!, model: "gpt-5.4" }, + }; + await client().generateCompletion(params({ messages })); + messages[1] = { + ...messages[1], + providerState: { ...first.providerState!, providerType: "anthropic" }, + }; + await client().generateCompletion(params({ messages })); + for (const call of [0, 1]) { + const types = (sentBody(mock, call).input as Array<{ type: string }>).map( + (i) => i.type, + ); + expect(types).not.toContain("reasoning"); + } + }); +}); + +describe("Codex reasoning refusals", () => { + it("retries without the summary the backend refused, and remembers", async () => { + const mock = vi + .fn() + .mockResolvedValueOnce( + errorResponse(400, fixture("codex-summary-rejected.json")), + ) + .mockImplementation(async () => + sseResponse(fixture("codex-summary.sse")), + ); + vi.stubGlobal("fetch", mock); + const c = client(); + const result = await c.generateCompletion( + params({ reasoning: { effort: "medium", summaries: true } }), + ); + expect(result.content).toBe( + "The train was moving for 3 hours and 10 minutes.", + ); + expect(sentBody(mock, 0).reasoning).toEqual({ + effort: "medium", + summary: "auto", + }); + expect(sentBody(mock, 1).reasoning).toEqual({ effort: "medium" }); + // Same level, only the summary went: not a downgrade. + expect(result.reasoning?.downgradedTo).toBeUndefined(); + + await c.generateCompletion( + params({ reasoning: { effort: "medium", summaries: true } }), + ); + expect(mock).toHaveBeenCalledTimes(3); + expect(sentBody(mock, 2).reasoning).toEqual({ effort: "medium" }); + }); + + it("steps down to the nearest effort the slug names, and remembers", async () => { + // The recorded refusal, with the slug's list narrowed to below "high". + const refusal = fixture("codex-effort-rejected.json") + .replace("'minimal'", "'high'") + .replace( + "'none', 'low', 'medium', 'high', and 'xhigh'", + "'none', 'low', and 'medium'", + ); + const mock = vi + .fn() + .mockResolvedValueOnce(errorResponse(400, refusal)) + .mockImplementation(async () => + sseResponse(fixture("codex-summary.sse")), + ); + vi.stubGlobal("fetch", mock); + const c = client(); + const result = await c.generateCompletion( + params({ reasoning: { effort: "high", summaries: true } }), + ); + expect(sentBody(mock, 1).reasoning).toEqual({ + effort: "medium", + summary: "auto", + }); + expect(result.reasoning?.downgradedTo).toBe("medium"); + + const again = await c.generateCompletion( + params({ reasoning: { effort: "high", summaries: true } }), + ); + expect(mock).toHaveBeenCalledTimes(3); + expect(sentBody(mock, 2).reasoning.effort).toBe("medium"); + expect(again.reasoning?.downgradedTo).toBe("medium"); + }); + + it("drops reasoning entirely when the refusal names no alternative", async () => { + const mock = vi + .fn() + .mockResolvedValueOnce( + errorResponse( + 400, + JSON.stringify({ + error: { + message: "Reasoning is not supported for this model.", + type: "invalid_request_error", + param: "reasoning", + }, + }), + ), + ) + .mockImplementation(async () => + sseResponse(fixture("codex-after-tool.sse")), + ); + vi.stubGlobal("fetch", mock); + const result = await client().generateCompletion( + params({ reasoning: { effort: "low", summaries: false } }), + ); + expect(sentBody(mock, 1).reasoning).toBeNull(); + expect(sentBody(mock, 1).include).toEqual([]); + expect(result.reasoning?.downgradedTo).toBe("off"); + expect(result.content).toBe("Paris weather: 18°C, cloudy."); + }); + + it("does not learn from a 400 that is not about reasoning", async () => { + const mock = vi.fn().mockResolvedValue( + errorResponse( + 400, + JSON.stringify({ + error: { message: "Invalid schema for function 'get_weather'." }, + }), + ), + ); + vi.stubGlobal("fetch", mock); + await expect( + client().generateCompletion( + params({ reasoning: { effort: "low", summaries: true } }), + ), + ).rejects.toMatchObject({ httpStatus: 400 }); + expect(mock).toHaveBeenCalledTimes(1); + }); +}); diff --git a/src/lib/ai/__tests__/fixtures/reasoning/codex-after-tool.sse b/src/lib/ai/__tests__/fixtures/reasoning/codex-after-tool.sse new file mode 100644 index 000000000..391ec2cd8 --- /dev/null +++ b/src/lib/ai/__tests__/fixtures/reasoning/codex-after-tool.sse @@ -0,0 +1,51 @@ +event: response.created +data: {"type":"response.created","response":{"id":"resp_03743ca81429e023016ac52524c4c487d284b7ab8375c47930","object":"response","created_at":1791304996,"status":"in_progress","access_programs":null,"background":false,"completed_at":null,"error":null,"frequency_penalty":0,"incomplete_details":null,"max_output_tokens":null,"max_tool_calls":null,"model":"gpt-5.5","moderation":null,"output":[],"parallel_tool_calls":false,"presence_penalty":0,"previous_response_id":null,"prompt_cache_retention":"24h","reasoning":{"context":"current_turn","effort":"low","mode":"standard","summary":"detailed"},"service_tier":"auto","store":false,"temperature":1,"text":{"format":{"type":"text"},"verbosity":"medium"},"tool_choice":"auto","top_logprobs":0,"top_p":0.98,"truncation":"disabled","usage":null,"user":null,"metadata":{}},"sequence_number":0} + +event: response.in_progress +data: {"type":"response.in_progress","response":{"id":"resp_03743ca81429e023016ac52524c4c487d284b7ab8375c47930","object":"response","created_at":1791304996,"status":"in_progress","access_programs":null,"background":false,"completed_at":null,"error":null,"frequency_penalty":0,"incomplete_details":null,"max_output_tokens":null,"max_tool_calls":null,"model":"gpt-5.5","moderation":null,"output":[],"parallel_tool_calls":false,"presence_penalty":0,"previous_response_id":null,"prompt_cache_retention":"24h","reasoning":{"context":"current_turn","effort":"low","mode":"standard","summary":"detailed"},"service_tier":"auto","store":false,"temperature":1,"text":{"format":{"type":"text"},"verbosity":"medium"},"tool_choice":"auto","top_logprobs":0,"top_p":0.98,"truncation":"disabled","usage":null,"user":null,"metadata":{}},"sequence_number":1} + +event: response.output_item.added +data: {"type":"response.output_item.added","item":{"id":"msg_03743ca81429e023016ac52525a89c87d280694ca111c7d02e","type":"message","status":"in_progress","content":[],"phase":"final_answer","role":"assistant"},"output_index":0,"sequence_number":2} + +event: response.content_part.added +data: {"type":"response.content_part.added","content_index":0,"item_id":"msg_03743ca81429e023016ac52525a89c87d280694ca111c7d02e","output_index":0,"part":{"type":"output_text","annotations":[],"logprobs":[],"text":""},"sequence_number":3} + +event: response.output_text.delta +data: {"type":"response.output_text.delta","content_index":0,"delta":"Paris","item_id":"msg_03743ca81429e023016ac52525a89c87d280694ca111c7d02e","logprobs":[],"output_index":0,"sequence_number":4} + +event: response.output_text.delta +data: {"type":"response.output_text.delta","content_index":0,"delta":" weather","item_id":"msg_03743ca81429e023016ac52525a89c87d280694ca111c7d02e","logprobs":[],"output_index":0,"sequence_number":5} + +event: response.output_text.delta +data: {"type":"response.output_text.delta","content_index":0,"delta":":","item_id":"msg_03743ca81429e023016ac52525a89c87d280694ca111c7d02e","logprobs":[],"output_index":0,"sequence_number":6} + +event: response.output_text.delta +data: {"type":"response.output_text.delta","content_index":0,"delta":" ","item_id":"msg_03743ca81429e023016ac52525a89c87d280694ca111c7d02e","logprobs":[],"output_index":0,"sequence_number":7} + +event: response.output_text.delta +data: {"type":"response.output_text.delta","content_index":0,"delta":"18","item_id":"msg_03743ca81429e023016ac52525a89c87d280694ca111c7d02e","logprobs":[],"output_index":0,"sequence_number":8} + +event: response.output_text.delta +data: {"type":"response.output_text.delta","content_index":0,"delta":"°C","item_id":"msg_03743ca81429e023016ac52525a89c87d280694ca111c7d02e","logprobs":[],"output_index":0,"sequence_number":9} + +event: response.output_text.delta +data: {"type":"response.output_text.delta","content_index":0,"delta":",","item_id":"msg_03743ca81429e023016ac52525a89c87d280694ca111c7d02e","logprobs":[],"output_index":0,"sequence_number":10} + +event: response.output_text.delta +data: {"type":"response.output_text.delta","content_index":0,"delta":" cloudy","item_id":"msg_03743ca81429e023016ac52525a89c87d280694ca111c7d02e","logprobs":[],"output_index":0,"sequence_number":11} + +event: response.output_text.delta +data: {"type":"response.output_text.delta","content_index":0,"delta":".","item_id":"msg_03743ca81429e023016ac52525a89c87d280694ca111c7d02e","logprobs":[],"output_index":0,"sequence_number":12} + +event: response.output_text.done +data: {"type":"response.output_text.done","content_index":0,"item_id":"msg_03743ca81429e023016ac52525a89c87d280694ca111c7d02e","logprobs":[],"output_index":0,"sequence_number":13,"text":"Paris weather: 18°C, cloudy."} + +event: response.content_part.done +data: {"type":"response.content_part.done","content_index":0,"item_id":"msg_03743ca81429e023016ac52525a89c87d280694ca111c7d02e","output_index":0,"part":{"type":"output_text","annotations":[],"logprobs":[],"text":"Paris weather: 18°C, cloudy."},"sequence_number":14} + +event: response.output_item.done +data: {"type":"response.output_item.done","item":{"id":"msg_03743ca81429e023016ac52525a89c87d280694ca111c7d02e","type":"message","status":"completed","content":[{"type":"output_text","annotations":[],"logprobs":[],"text":"Paris weather: 18°C, cloudy."}],"phase":"final_answer","role":"assistant"},"output_index":0,"sequence_number":15} + +event: response.completed +data: {"type":"response.completed","response":{"id":"resp_03743ca81429e023016ac52524c4c487d284b7ab8375c47930","object":"response","created_at":1791304996,"status":"completed","access_programs":null,"background":false,"completed_at":1791304997,"error":null,"frequency_penalty":0,"incomplete_details":null,"max_output_tokens":null,"max_tool_calls":null,"model":"gpt-5.5","moderation":null,"output":[],"parallel_tool_calls":false,"presence_penalty":0,"previous_response_id":null,"prompt_cache_retention":"24h","reasoning":{"context":"current_turn","effort":"low","mode":"standard","summary":"detailed"},"service_tier":"default","store":false,"temperature":1,"text":{"format":{"type":"text"},"verbosity":"medium"},"tool_choice":"auto","top_logprobs":0,"top_p":0.98,"truncation":"disabled","usage":{"input_tokens":205,"input_tokens_details":{"cache_write_tokens":0,"cached_tokens":0},"output_tokens":13,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":218},"user":null,"metadata":{}},"sequence_number":16} + diff --git a/src/lib/ai/__tests__/fixtures/reasoning/codex-effort-rejected.json b/src/lib/ai/__tests__/fixtures/reasoning/codex-effort-rejected.json new file mode 100644 index 000000000..f000a5253 --- /dev/null +++ b/src/lib/ai/__tests__/fixtures/reasoning/codex-effort-rejected.json @@ -0,0 +1,8 @@ +{ + "error": { + "message": "Unsupported value: 'minimal' is not supported with the 'gpt-5.5' model. Supported values are: 'none', 'low', 'medium', 'high', and 'xhigh'.", + "type": "invalid_request_error", + "param": "reasoning.effort", + "code": "unsupported_value" + } +} diff --git a/src/lib/ai/__tests__/fixtures/reasoning/codex-summary-rejected.json b/src/lib/ai/__tests__/fixtures/reasoning/codex-summary-rejected.json new file mode 100644 index 000000000..4b15d9294 --- /dev/null +++ b/src/lib/ai/__tests__/fixtures/reasoning/codex-summary-rejected.json @@ -0,0 +1,8 @@ +{ + "error": { + "message": "Invalid value: 'verbose'. Supported values are: 'concise', 'detailed', and 'auto'.", + "type": "invalid_request_error", + "param": "reasoning.summary", + "code": "invalid_value" + } +} diff --git a/src/lib/ai/__tests__/fixtures/reasoning/codex-summary.sse b/src/lib/ai/__tests__/fixtures/reasoning/codex-summary.sse new file mode 100644 index 000000000..6e9c6c616 --- /dev/null +++ b/src/lib/ai/__tests__/fixtures/reasoning/codex-summary.sse @@ -0,0 +1,81 @@ +event: response.created +data: {"type":"response.created","sequence_number":0,"response":{"id":"resp_01241f364fbd794f016ac5253e2ce08191afe0c9a23644957f","object":"response","created_at":1791305022,"completed_at":null,"status":"in_progress","background":false,"frequency_penalty":0,"presence_penalty":0,"moderation":null,"model":"gpt-5.5","access_programs":null,"previous_response_id":null,"output":[],"error":null,"incomplete_details":null,"tool_choice":"auto","parallel_tool_calls":false,"prompt_cache_retention":"24h","text":{"format":{"type":"text"},"verbosity":"medium"},"top_p":0.98,"top_logprobs":0,"temperature":1,"reasoning":{"effort":"medium","summary":"detailed","context":"current_turn","mode":"standard"},"user":null,"usage":null,"store":false,"metadata":{},"max_output_tokens":null,"max_tool_calls":null,"service_tier":"auto","truncation":"disabled"}} + +event: response.in_progress +data: {"type":"response.in_progress","sequence_number":1,"response":{"id":"resp_01241f364fbd794f016ac5253e2ce08191afe0c9a23644957f","object":"response","created_at":1791305022,"completed_at":null,"status":"in_progress","background":false,"frequency_penalty":0,"presence_penalty":0,"moderation":null,"model":"gpt-5.5","access_programs":null,"previous_response_id":null,"output":[],"error":null,"incomplete_details":null,"tool_choice":"auto","parallel_tool_calls":false,"prompt_cache_retention":"24h","text":{"format":{"type":"text"},"verbosity":"medium"},"top_p":0.98,"top_logprobs":0,"temperature":1,"reasoning":{"effort":"medium","summary":"detailed","context":"current_turn","mode":"standard"},"user":null,"usage":null,"store":false,"metadata":{},"max_output_tokens":null,"max_tool_calls":null,"service_tier":"auto","truncation":"disabled"}} + +event: response.output_item.added +data: {"type":"response.output_item.added","sequence_number":2,"output_index":0,"item":{"type":"reasoning","id":"rs_01241f364fbd794f016ac5253e96a081919fae3ca9ea0bea08","summary":[],"content":[],"encrypted_content":"enc-fixture-1"}} + +event: response.reasoning_summary_part.added +data: {"type":"response.reasoning_summary_part.added","sequence_number":3,"item_id":"rs_01241f364fbd794f016ac5253e96a081919fae3ca9ea0bea08","output_index":0,"summary_index":0,"part":{"type":"summary_text","text":""}} + +event: response.reasoning_summary_text.delta +data: {"type":"response.reasoning_summary_text.delta","sequence_number":4,"item_id":"rs_01241f364fbd794f016ac5253e96a081919fae3ca9ea0bea08","output_index":0,"summary_index":0,"delta":"**Calculating elapsed time minus offset**"} + +event: response.reasoning_summary_text.done +data: {"type":"response.reasoning_summary_text.done","sequence_number":5,"item_id":"rs_01241f364fbd794f016ac5253e96a081919fae3ca9ea0bea08","output_index":0,"summary_index":0,"text":"**Calculating elapsed time minus offset**"} + +event: response.reasoning_summary_part.done +data: {"type":"response.reasoning_summary_part.done","sequence_number":6,"item_id":"rs_01241f364fbd794f016ac5253e96a081919fae3ca9ea0bea08","output_index":0,"summary_index":0,"part":{"type":"summary_text","text":"**Calculating elapsed time minus offset**"}} + +event: response.output_item.done +data: {"type":"response.output_item.done","sequence_number":7,"output_index":0,"item":{"type":"reasoning","id":"rs_01241f364fbd794f016ac5253e96a081919fae3ca9ea0bea08","summary":[{"type":"summary_text","text":"**Calculating elapsed time minus offset**"}],"content":[],"encrypted_content":"enc-fixture-2"}} + +event: response.output_item.added +data: {"type":"response.output_item.added","sequence_number":8,"output_index":1,"item":{"type":"message","id":"msg_01241f364fbd794f016ac5253f35e4819197bcd3f7451f26aa","role":"assistant","status":"in_progress","content":[],"phase":"final_answer"}} + +event: response.content_part.added +data: {"type":"response.content_part.added","sequence_number":9,"item_id":"msg_01241f364fbd794f016ac5253f35e4819197bcd3f7451f26aa","output_index":1,"content_index":0,"part":{"type":"output_text","text":"","annotations":[],"logprobs":[]}} + +event: response.output_text.delta +data: {"type":"response.output_text.delta","sequence_number":10,"item_id":"msg_01241f364fbd794f016ac5253f35e4819197bcd3f7451f26aa","output_index":1,"content_index":0,"delta":"The","logprobs":[]} + +event: response.output_text.delta +data: {"type":"response.output_text.delta","sequence_number":11,"item_id":"msg_01241f364fbd794f016ac5253f35e4819197bcd3f7451f26aa","output_index":1,"content_index":0,"delta":" train","logprobs":[]} + +event: response.output_text.delta +data: {"type":"response.output_text.delta","sequence_number":12,"item_id":"msg_01241f364fbd794f016ac5253f35e4819197bcd3f7451f26aa","output_index":1,"content_index":0,"delta":" was","logprobs":[]} + +event: response.output_text.delta +data: {"type":"response.output_text.delta","sequence_number":13,"item_id":"msg_01241f364fbd794f016ac5253f35e4819197bcd3f7451f26aa","output_index":1,"content_index":0,"delta":" moving","logprobs":[]} + +event: response.output_text.delta +data: {"type":"response.output_text.delta","sequence_number":14,"item_id":"msg_01241f364fbd794f016ac5253f35e4819197bcd3f7451f26aa","output_index":1,"content_index":0,"delta":" for","logprobs":[]} + +event: response.output_text.delta +data: {"type":"response.output_text.delta","sequence_number":15,"item_id":"msg_01241f364fbd794f016ac5253f35e4819197bcd3f7451f26aa","output_index":1,"content_index":0,"delta":" ","logprobs":[]} + +event: response.output_text.delta +data: {"type":"response.output_text.delta","sequence_number":16,"item_id":"msg_01241f364fbd794f016ac5253f35e4819197bcd3f7451f26aa","output_index":1,"content_index":0,"delta":"3","logprobs":[]} + +event: response.output_text.delta +data: {"type":"response.output_text.delta","sequence_number":17,"item_id":"msg_01241f364fbd794f016ac5253f35e4819197bcd3f7451f26aa","output_index":1,"content_index":0,"delta":" hours","logprobs":[]} + +event: response.output_text.delta +data: {"type":"response.output_text.delta","sequence_number":18,"item_id":"msg_01241f364fbd794f016ac5253f35e4819197bcd3f7451f26aa","output_index":1,"content_index":0,"delta":" and","logprobs":[]} + +event: response.output_text.delta +data: {"type":"response.output_text.delta","sequence_number":19,"item_id":"msg_01241f364fbd794f016ac5253f35e4819197bcd3f7451f26aa","output_index":1,"content_index":0,"delta":" ","logprobs":[]} + +event: response.output_text.delta +data: {"type":"response.output_text.delta","sequence_number":20,"item_id":"msg_01241f364fbd794f016ac5253f35e4819197bcd3f7451f26aa","output_index":1,"content_index":0,"delta":"10","logprobs":[]} + +event: response.output_text.delta +data: {"type":"response.output_text.delta","sequence_number":21,"item_id":"msg_01241f364fbd794f016ac5253f35e4819197bcd3f7451f26aa","output_index":1,"content_index":0,"delta":" minutes","logprobs":[]} + +event: response.output_text.delta +data: {"type":"response.output_text.delta","sequence_number":22,"item_id":"msg_01241f364fbd794f016ac5253f35e4819197bcd3f7451f26aa","output_index":1,"content_index":0,"delta":".","logprobs":[]} + +event: response.output_text.done +data: {"type":"response.output_text.done","sequence_number":23,"item_id":"msg_01241f364fbd794f016ac5253f35e4819197bcd3f7451f26aa","output_index":1,"content_index":0,"text":"The train was moving for 3 hours and 10 minutes.","logprobs":[]} + +event: response.content_part.done +data: {"type":"response.content_part.done","sequence_number":24,"item_id":"msg_01241f364fbd794f016ac5253f35e4819197bcd3f7451f26aa","output_index":1,"content_index":0,"part":{"type":"output_text","text":"The train was moving for 3 hours and 10 minutes.","annotations":[],"logprobs":[]}} + +event: response.output_item.done +data: {"type":"response.output_item.done","sequence_number":25,"output_index":1,"item":{"type":"message","id":"msg_01241f364fbd794f016ac5253f35e4819197bcd3f7451f26aa","role":"assistant","status":"completed","content":[{"type":"output_text","text":"The train was moving for 3 hours and 10 minutes.","annotations":[],"logprobs":[]}],"phase":"final_answer"}} + +event: response.completed +data: {"type":"response.completed","sequence_number":26,"response":{"id":"resp_01241f364fbd794f016ac5253e2ce08191afe0c9a23644957f","object":"response","created_at":1791305022,"completed_at":1791305023,"status":"completed","background":false,"frequency_penalty":0,"presence_penalty":0,"moderation":null,"model":"gpt-5.5","access_programs":null,"previous_response_id":null,"output":[{"type":"reasoning","id":"rs_01241f364fbd794f016ac5253e96a081919fae3ca9ea0bea08","summary":[{"type":"summary_text","text":"**Calculating elapsed time minus offset**"}],"content":[],"encrypted_content":"enc-fixture-3"},{"type":"message","id":"msg_01241f364fbd794f016ac5253f35e4819197bcd3f7451f26aa","role":"assistant","status":"completed","content":[{"type":"output_text","text":"The train was moving for 3 hours and 10 minutes.","annotations":[],"logprobs":[]}],"phase":"final_answer"}],"error":null,"incomplete_details":null,"tool_choice":"auto","parallel_tool_calls":false,"prompt_cache_retention":"24h","text":{"format":{"type":"text"},"verbosity":"medium"},"top_p":0.98,"top_logprobs":0,"temperature":1,"reasoning":{"effort":"medium","summary":"detailed","context":"current_turn","mode":"standard"},"user":null,"usage":{"input_tokens":49,"input_tokens_details":{"cached_tokens":0,"cache_write_tokens":0},"output_tokens":52,"output_tokens_details":{"reasoning_tokens":33},"total_tokens":101},"store":false,"metadata":{},"max_output_tokens":null,"max_tool_calls":null,"service_tier":"default","truncation":"disabled"}} + diff --git a/src/lib/ai/__tests__/fixtures/reasoning/codex-tool-round.sse b/src/lib/ai/__tests__/fixtures/reasoning/codex-tool-round.sse new file mode 100644 index 000000000..5b723cd08 --- /dev/null +++ b/src/lib/ai/__tests__/fixtures/reasoning/codex-tool-round.sse @@ -0,0 +1,39 @@ +event: response.created +data: {"type":"response.created","response":{"id":"resp_03743ca81429e023016ac5252240a887d29b3b3e712af57529","object":"response","created_at":1791304994,"status":"in_progress","access_programs":null,"background":false,"completed_at":null,"error":null,"frequency_penalty":0,"incomplete_details":null,"max_output_tokens":null,"max_tool_calls":null,"model":"gpt-5.5","moderation":null,"output":[],"parallel_tool_calls":false,"presence_penalty":0,"previous_response_id":null,"prompt_cache_retention":"24h","reasoning":{"context":"current_turn","effort":"low","mode":"standard","summary":"detailed"},"service_tier":"auto","store":false,"temperature":1,"text":{"format":{"type":"text"},"verbosity":"medium"},"tool_choice":"auto","top_logprobs":0,"top_p":0.98,"truncation":"disabled","usage":null,"user":null,"metadata":{}},"sequence_number":0} + +event: response.in_progress +data: {"type":"response.in_progress","response":{"id":"resp_03743ca81429e023016ac5252240a887d29b3b3e712af57529","object":"response","created_at":1791304994,"status":"in_progress","access_programs":null,"background":false,"completed_at":null,"error":null,"frequency_penalty":0,"incomplete_details":null,"max_output_tokens":null,"max_tool_calls":null,"model":"gpt-5.5","moderation":null,"output":[],"parallel_tool_calls":false,"presence_penalty":0,"previous_response_id":null,"prompt_cache_retention":"24h","reasoning":{"context":"current_turn","effort":"low","mode":"standard","summary":"detailed"},"service_tier":"auto","store":false,"temperature":1,"text":{"format":{"type":"text"},"verbosity":"medium"},"tool_choice":"auto","top_logprobs":0,"top_p":0.98,"truncation":"disabled","usage":null,"user":null,"metadata":{}},"sequence_number":1} + +event: response.output_item.added +data: {"type":"response.output_item.added","item":{"id":"rs_03743ca81429e023016ac52522f51c87d297ef373ae4b945c9","type":"reasoning","content":[],"encrypted_content":"enc-fixture-4","summary":[]},"output_index":0,"sequence_number":2} + +event: response.output_item.done +data: {"type":"response.output_item.done","item":{"id":"rs_03743ca81429e023016ac52522f51c87d297ef373ae4b945c9","type":"reasoning","content":[],"encrypted_content":"enc-fixture-5","summary":[]},"output_index":0,"sequence_number":3} + +event: response.output_item.added +data: {"type":"response.output_item.added","item":{"id":"fc_03743ca81429e023016ac52523984487d2b91da4a24fe34ece","type":"function_call","status":"in_progress","arguments":"","call_id":"call_edjM3le4GcXAsF7vywK4kQgY","name":"get_weather"},"output_index":1,"sequence_number":4} + +event: response.function_call_arguments.delta +data: {"type":"response.function_call_arguments.delta","delta":"{\"","item_id":"fc_03743ca81429e023016ac52523984487d2b91da4a24fe34ece","output_index":1,"sequence_number":5} + +event: response.function_call_arguments.delta +data: {"type":"response.function_call_arguments.delta","delta":"city","item_id":"fc_03743ca81429e023016ac52523984487d2b91da4a24fe34ece","output_index":1,"sequence_number":6} + +event: response.function_call_arguments.delta +data: {"type":"response.function_call_arguments.delta","delta":"\":\"","item_id":"fc_03743ca81429e023016ac52523984487d2b91da4a24fe34ece","output_index":1,"sequence_number":7} + +event: response.function_call_arguments.delta +data: {"type":"response.function_call_arguments.delta","delta":"Paris","item_id":"fc_03743ca81429e023016ac52523984487d2b91da4a24fe34ece","output_index":1,"sequence_number":8} + +event: response.function_call_arguments.delta +data: {"type":"response.function_call_arguments.delta","delta":"\"}","item_id":"fc_03743ca81429e023016ac52523984487d2b91da4a24fe34ece","output_index":1,"sequence_number":9} + +event: response.function_call_arguments.done +data: {"type":"response.function_call_arguments.done","arguments":"{\"city\":\"Paris\"}","item_id":"fc_03743ca81429e023016ac52523984487d2b91da4a24fe34ece","output_index":1,"sequence_number":10} + +event: response.output_item.done +data: {"type":"response.output_item.done","item":{"id":"fc_03743ca81429e023016ac52523984487d2b91da4a24fe34ece","type":"function_call","status":"completed","arguments":"{\"city\":\"Paris\"}","call_id":"call_edjM3le4GcXAsF7vywK4kQgY","name":"get_weather"},"output_index":1,"sequence_number":11} + +event: response.completed +data: {"type":"response.completed","response":{"id":"resp_03743ca81429e023016ac5252240a887d29b3b3e712af57529","object":"response","created_at":1791304994,"status":"completed","access_programs":null,"background":false,"completed_at":1791304995,"error":null,"frequency_penalty":0,"incomplete_details":null,"max_output_tokens":null,"max_tool_calls":null,"model":"gpt-5.5","moderation":null,"output":[],"parallel_tool_calls":false,"presence_penalty":0,"previous_response_id":null,"prompt_cache_retention":"24h","reasoning":{"context":"current_turn","effort":"low","mode":"standard","summary":"detailed"},"service_tier":"default","store":false,"temperature":1,"text":{"format":{"type":"text"},"verbosity":"medium"},"tool_choice":"auto","top_logprobs":0,"top_p":0.98,"truncation":"disabled","usage":{"input_tokens":147,"input_tokens_details":{"cache_write_tokens":0,"cached_tokens":0},"output_tokens":36,"output_tokens_details":{"reasoning_tokens":16},"total_tokens":183},"user":null,"metadata":{}},"sequence_number":12} + diff --git a/src/lib/ai/codex-client.ts b/src/lib/ai/codex-client.ts index 5cba79686..863ad36ac 100644 --- a/src/lib/ai/codex-client.ts +++ b/src/lib/ai/codex-client.ts @@ -5,11 +5,19 @@ import type { AIProvider, AiContentPart, AiMessage, + AiProviderState, AiToolCall, AiToolDef, CompletionParams, CompletionResult, } from "./types"; +import { + annotateReasoningDowngrade, + learnedReasoningDialect, + rememberReasoningDialect, +} from "./reasoning/dialect-cache"; +import type { ReasoningLevel } from "./reasoning/levels"; +import { levelOfWireEffort, reasoningTitleOf } from "./reasoning/support"; import { getCachedCodexSlug, invalidateCachedCodexSlug, @@ -140,7 +148,10 @@ type CodexInputItem = type: "function_call_output"; call_id: string; output: string; - }; + } + // v1.41 — a reasoning item from an earlier round of the same turn, handed + // back exactly as the stream delivered it (encrypted content included). + | Record; /** * Map an `AiContentPart[]` body into Responses input content blocks. Text parts @@ -175,7 +186,10 @@ function mapCodexParts( * top-level items the Responses API expects; everything else is a `message`. * A text-only single user turn is byte-identical to the pre-refactor wire. */ -function buildCodexInput(messages: AiMessage[]): CodexInputItem[] { +function buildCodexInput( + messages: AiMessage[], + slug: string, +): CodexInputItem[] { const items: CodexInputItem[] = []; for (const m of messages) { if (m.role === "tool") { @@ -198,6 +212,18 @@ function buildCodexInput(messages: AiMessage[]): CodexInputItem[] { typeof m.content === "string" ? [{ type: textType, text: m.content }] : mapCodexParts(m.content, m.role); + // v1.41 — the reasoning items this round produced go back before its + // message and function calls, unchanged: with `store: false` the server + // keeps nothing, and a tool round that loses its reasoning items reasons + // from scratch (https://developers.openai.com/cookbook/examples/responses_api/reasoning_items). + // Only state this client produced for this slug is re-inserted; anything + // else (another provider's state, a slug the chain walked away from) is + // dropped, because an encrypted item is only readable where it was made. + if (m.role === "assistant") { + for (const item of codexStateItems(m.providerState, slug)) { + items.push(item); + } + } items.push({ type: "message", role: m.role, content }); if (m.role === "assistant" && m.toolCalls) { for (const tc of m.toolCalls) { @@ -213,6 +239,149 @@ function buildCodexInput(messages: AiMessage[]): CodexInputItem[] { return items; } +/** The reasoning items of `state` when this client made it for `slug`. */ +function codexStateItems( + state: AiProviderState | undefined, + slug: string, +): Record[] { + if (!state || state.providerType !== "codex" || state.model !== slug) { + return []; + } + return state.items.filter( + (item): item is Record => + item !== null && + typeof item === "object" && + (item as { type?: unknown }).type === "reasoning", + ); +} + +// ── v1.41 reasoning wire ──────────────────────────────────────────────────── + +/** + * The Responses-API efforts in ascending order. The Codex backend names the + * values a slug accepts in its 400 body ("Supported values are: 'none', + * 'low', …"), which is how a slug's ladder is learned. + */ +const CODEX_EFFORT_ORDER = [ + "none", + "minimal", + "low", + "medium", + "high", + "xhigh", +] as const; + +/** What one request sends: `null` is today's `reasoning: null`. */ +type CodexReasoningWire = { effort: string; summary: boolean } | null; + +/** What a slug was learned to refuse (see `reasoning/dialect-cache.ts`). */ +interface CodexReasoningDialect { + /** The slug refused `reasoning` altogether. */ + disabled?: true; + /** The slug refused `reasoning.summary`. */ + noSummary?: true; + /** The efforts the slug named as supported when it refused one. */ + efforts?: string[]; +} + +function dialectKey(slug: string) { + return { provider: "codex", endpoint: CODEX_ENDPOINT, model: slug }; +} + +function readCodexDialect(slug: string): CodexReasoningDialect { + const raw = learnedReasoningDialect(dialectKey(slug)); + if (!raw) return {}; + try { + return JSON.parse(raw) as CodexReasoningDialect; + } catch { + return {}; + } +} + +function wantedEffort(level: ReasoningLevel): string { + // "Off" is a real off on the current slugs: a 2026-10-06 probe answered + // `effort: "none"` on gpt-5.5 with a non-reasoning reply. A slug that + // refuses it learns its lowest supported value instead. + return level === "off" ? "none" : level; +} + +/** The supported effort closest to `wanted` from below, else the lowest. */ +function nearestEffort(wanted: string, supported: string[]): string | null { + const rank = (e: string) => + (CODEX_EFFORT_ORDER as readonly string[]).indexOf(e); + const known = supported + .filter((e) => rank(e) >= 0) + .sort((a, b) => rank(a) - rank(b)); + if (known.length === 0) return null; + const below = known.filter((e) => rank(e) <= rank(wanted)); + return below.length > 0 ? below[below.length - 1] : known[0]; +} + +function planCodexReasoning( + reasoning: CompletionParams["reasoning"], + slug: string, +): CodexReasoningWire { + if (!reasoning) return null; + const dialect = readCodexDialect(slug); + if (dialect.disabled) return null; + let effort: string | null = wantedEffort(reasoning.effort); + if (dialect.efforts && !dialect.efforts.includes(effort)) { + effort = nearestEffort(effort, dialect.efforts); + } + if (!effort) return null; + return { + effort, + summary: reasoning.summaries && effort !== "none" && !dialect.noSummary, + }; +} + +/** + * Read a 400 that refuses a reasoning parameter and learn what the slug + * accepts instead. Returns the lesser wire to retry with, or `undefined` + * when the body is not about reasoning (the caller's error path takes it). + */ +function learnFromReasoningRejection( + slug: string, + wire: Exclude, + status: number, + body: string, +): CodexReasoningWire | undefined { + // Only a body that names the reasoning parameter counts; a 400 about the + // prompt or a tool must not teach the slug anything. + if (status !== 400 || !/reasoning/i.test(body)) { + return undefined; + } + const dialect = readCodexDialect(slug); + let next: CodexReasoningWire; + if (wire.summary && /reasoning\.summary|summary/i.test(body)) { + dialect.noSummary = true; + next = { effort: wire.effort, summary: false }; + } else { + const listed = /supported values are:([^"\n]*)/i.exec(body); + const efforts = listed + ? Array.from(listed[1].matchAll(/'([a-z]+)'/g), (m) => m[1]) + : []; + const nearest = + efforts.length > 0 && !efforts.includes(wire.effort) + ? nearestEffort(wire.effort, efforts) + : null; + if (nearest) { + dialect.efforts = efforts; + next = { effort: nearest, summary: wire.summary && nearest !== "none" }; + } else { + dialect.disabled = true; + next = null; + } + } + rememberReasoningDialect(dialectKey(slug), JSON.stringify(dialect)); + annotateReasoningDowngrade( + "codex", + wire.effort, + next ? next.effort : "default", + ); + return next; +} + /** * Map tool defs into the Responses `function` tool shape. * @@ -311,7 +480,8 @@ export class CodexClient implements AIProvider { for (const slug of order) { attempted.push(slug); - const firstAttempt = await this.doRequest(params, slug); + let wire = planCodexReasoning(params.reasoning, slug); + const firstAttempt = await this.doRequest(params, slug, wire); if (firstAttempt.status === 401) { // Token-refresh path — don't walk the chain, this is auth-state @@ -319,7 +489,7 @@ export class CodexClient implements AIProvider { const fresh = await this.onTokenRefresh(); this.accessToken = fresh.accessToken; this.accountId = fresh.accountId; - const retryAfterRefresh = await this.doRequest(params, slug); + const retryAfterRefresh = await this.doRequest(params, slug, wire); if (retryAfterRefresh.ok) { this.lastDiagnostics = { attempted, @@ -327,7 +497,7 @@ export class CodexClient implements AIProvider { workingSlug: slug, }; setCachedCodexSlug(slug); - return this.parseResponse(retryAfterRefresh, slug); + return this.parseResponse(retryAfterRefresh, slug, params, wire); } // Even after refresh — surface the auth failure verbatim, do // NOT walk the chain (auth issues don't get fixed by changing @@ -339,30 +509,51 @@ export class CodexClient implements AIProvider { ); } - if (firstAttempt.ok) { + let attempt = firstAttempt; + let rawBody = ""; + // v1.41 — a slug that refuses a reasoning parameter is asked again with + // the lesser request it names (no summary, a lower effort, no reasoning + // at all), and the slug remembers it. The answer is never lost to a + // reasoning refusal. Bounded: each step strictly reduces the request. + for (let step = 0; !attempt.ok && step < 3; step += 1) { + rawBody = await attempt.text().catch(() => ""); + if (!wire) break; + const next = learnFromReasoningRejection( + slug, + wire, + attempt.status, + redactBody(rawBody), + ); + if (next === undefined) break; + wire = next; + rawBody = ""; + attempt = await this.doRequest(params, slug, wire); + } + + if (attempt.ok) { this.lastDiagnostics = { attempted, cacheState, workingSlug: slug, }; setCachedCodexSlug(slug); - return this.parseResponse(firstAttempt, slug); + return this.parseResponse(attempt, slug, params, wire); } // Capture the body for the slug-rejection check. - const rawBody = await firstAttempt.text().catch(() => ""); + if (!rawBody) rawBody = await attempt.text().catch(() => ""); const bodyExcerpt = redactBody(rawBody); - if (isSlugRejection(firstAttempt.status, bodyExcerpt)) { + if (isSlugRejection(attempt.status, bodyExcerpt)) { // Slug rejected — drop the cache (in case it pointed here), // record the failure for diagnostics, and walk to the next // chain slot. invalidateCachedCodexSlug(); const slugErr = new Error( - `Codex slug "${slug}" rejected (${firstAttempt.status})`, + `Codex slug "${slug}" rejected (${attempt.status})`, ); Object.assign(slugErr, { - httpStatus: firstAttempt.status, + httpStatus: attempt.status, upstream: "codex", model: slug, bodyExcerpt, @@ -382,11 +573,11 @@ export class CodexClient implements AIProvider { // un-diagnosable. The body is already redacted of bearer/sk- secrets. const err = new Error( bodyExcerpt - ? `Codex request failed (${firstAttempt.status}): ${bodyExcerpt}` - : `Codex request failed (${firstAttempt.status})`, + ? `Codex request failed (${attempt.status}): ${bodyExcerpt}` + : `Codex request failed (${attempt.status})`, ); Object.assign(err, { - httpStatus: firstAttempt.status, + httpStatus: attempt.status, upstream: "codex", model: slug, bodyExcerpt, @@ -447,6 +638,7 @@ export class CodexClient implements AIProvider { private async doRequest( params: CompletionParams, slug: string, + wire: CodexReasoningWire, ): Promise { const sessionId = randomUUID(); const threadId = randomUUID(); @@ -466,7 +658,7 @@ export class CodexClient implements AIProvider { // v1.20.0 — `input` is now built from the full message array (multi-turn + // tool-result turns), with `tools` mapped to the Responses `function` shape // (the wire already declared `tools`/`tool_choice` — now they carry defs). - const input = buildCodexInput(params.messages); + const input = buildCodexInput(params.messages, slug); const tools = params.tools && params.tools.length > 0 ? buildCodexTools(params.tools) @@ -496,15 +688,28 @@ export class CodexClient implements AIProvider { tools, tool_choice: toolChoice, parallel_tool_calls: false, - // Reasoning is required-but-nullable on the wire. We don't ask - // for reasoning summaries — emit `null` so the JSON has the - // field present and the server stops complaining about a - // missing key (it returned 400 in earlier iterations). - reasoning: null, + // Reasoning is required-but-nullable on the wire. Without a + // requested level (every caller before v1.41, and every caller + // that does not opt in) it stays `null`, the field present so the + // server stops complaining about a missing key (it returned 400 in + // earlier iterations). With one, the official client's shape: + // `{ effort, summary }`, `summary` only when summaries are wanted. + // https://developers.openai.com/api/docs/guides/reasoning + reasoning: wire + ? { + effort: wire.effort, + ...(wire.summary ? { summary: "auto" } : {}), + } + : null, store: false, stream: true, - // Required field; empty array when not asking for reasoning. - include: [], + // Required field; empty array when not asking for reasoning. With + // reasoning on, the encrypted items come back so the next tool + // round can hand them over statelessly (`store` must stay false). + include: + wire && wire.effort !== "none" + ? ["reasoning.encrypted_content"] + : [], }), // SSE streaming completion — match the 60 s budget the other AI // clients use so a long generation is not clipped by the 15 s @@ -540,6 +745,8 @@ export class CodexClient implements AIProvider { private async parseResponse( res: Response, requestedSlug: string, + params: CompletionParams, + wire: CodexReasoningWire, ): Promise { if (!res.body) { // v1.20.1 — sentinel httpStatus + kind so the chain classifier can tell @@ -563,6 +770,31 @@ export class CodexClient implements AIProvider { let serverModel: string | null = null; const toolCalls: AiToolCall[] = []; + // v1.41 — reasoning. Events only reach `onReasoning` when the caller asked + // for reasoning (the contract in `types.ts`); summaries are model text and + // the receiver screens them. + const emit = params.reasoning ? params.onReasoning : undefined; + const reasoningItems: Record[] = []; + const summaries: string[] = []; + let reasoningTokens: number | null = null; + const summaryBuffers = new Map(); + const titled = new Set(); + const texted = new Set(); + const emitTitle = (key: string, text: string) => { + if (!emit || titled.has(key)) return; + const title = reasoningTitleOf(text); + if (title) { + titled.add(key); + emit({ kind: "title", text: title }); + } + }; + const emitText = (key: string, text: string) => { + if (!emit || texted.has(key) || !text.trim()) return; + emitTitle(key, text); + texted.add(key); + emit({ kind: "text", text }); + }; + // The server reports the actual routed model in the OpenAI-Model // response header — useful when safety-routing kicks in. const headerModel = res.headers.get("OpenAI-Model"); @@ -589,10 +821,14 @@ export class CodexClient implements AIProvider { let parsed: { type?: string; delta?: string; + text?: string; + item_id?: string; + summary_index?: number; item?: { type?: string; role?: string; content?: Array<{ type?: string; text?: string }>; + summary?: Array<{ type?: string; text?: string }>; id?: string; call_id?: string; name?: string; @@ -602,6 +838,7 @@ export class CodexClient implements AIProvider { usage?: { total_tokens?: number; input_tokens_details?: { cached_tokens?: number }; + output_tokens_details?: { reasoning_tokens?: number }; }; error?: { code?: string; message?: string }; incomplete_details?: { reason?: string }; @@ -619,6 +856,22 @@ export class CodexClient implements AIProvider { if (parsed.delta) deltaText += parsed.delta; break; } + // v1.41 — live reasoning summaries. A summary's first bold line + // is its title; it goes out as soon as it has closed, the full + // text when the part is done. + case "response.reasoning_summary_text.delta": { + const key = `${parsed.item_id ?? ""}:${parsed.summary_index ?? 0}`; + const buffered = + (summaryBuffers.get(key) ?? "") + (parsed.delta ?? ""); + summaryBuffers.set(key, buffered); + emitTitle(key, buffered); + break; + } + case "response.reasoning_summary_text.done": { + const key = `${parsed.item_id ?? ""}:${parsed.summary_index ?? 0}`; + emitText(key, parsed.text ?? summaryBuffers.get(key) ?? ""); + break; + } case "response.output_item.done": { const item = parsed.item; if (item?.type === "message" && item.role === "assistant") { @@ -627,6 +880,16 @@ export class CodexClient implements AIProvider { .map((c) => c.text!) .join(""); if (text) assembledText += text; + } else if (item?.type === "reasoning" && params.reasoning) { + // The whole item, untouched: the next round hands it back. + reasoningItems.push(item as Record); + (item.summary ?? []).forEach((part, index) => { + if (typeof part.text !== "string" || !part.text) return; + summaries.push(part.text); + // A part whose `.done` never arrived still reaches the + // receiver, once. + emitText(`${item.id ?? ""}:${index}`, part.text); + }); } else if (item?.type === "function_call") { // v1.20.0 — the model asked to call a tool. Surface it for // F1's loop; the id is the call_id the function_call_output @@ -644,6 +907,11 @@ export class CodexClient implements AIProvider { cachedInputTokens = parsed.response?.usage?.input_tokens_details?.cached_tokens ?? null; + // Reasoning tokens are billed as output and already inside + // `total_tokens`; reported separately for the trail and budget. + reasoningTokens = + parsed.response?.usage?.output_tokens_details + ?.reasoning_tokens ?? null; break; } case "response.failed": { @@ -713,6 +981,19 @@ export class CodexClient implements AIProvider { throw err; } + if (emit) emit({ kind: "done", text: "" }); + + const sentLevel = wire ? levelOfWireEffort(wire.effort) : null; + const reasoning: CompletionResult["reasoning"] = params.reasoning + ? { + summary: summaries, + tokens: reasoningTokens, + ...(sentLevel !== params.reasoning.effort + ? { downgradedTo: sentLevel ?? "off" } + : {}), + } + : undefined; + return { content, tokensUsed, @@ -721,6 +1002,18 @@ export class CodexClient implements AIProvider { providerType: "codex", ...(toolCalls.length > 0 ? { toolCalls } : {}), finishReason: toolCalls.length > 0 ? "tool_calls" : "stop", + ...(reasoning ? { reasoning } : {}), + // Keyed on the slug that was asked, not the routed model header: the + // next round re-inserts the items only when it asks the same slug. + ...(reasoningItems.length > 0 + ? { + providerState: { + providerType: "codex" as const, + model: requestedSlug, + items: reasoningItems, + }, + } + : {}), }; } } From 6b1bafca3c7064ae60cea870d2aacb3fd234c95c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Tue, 6 Oct 2026 19:26:29 +0200 Subject: [PATCH 3/7] feat(ai): ask Anthropic models to think in the dialect their generation takes Up to 4.5 a thinking budget (with the interleaved-thinking header on tool rounds), from 4.6 adaptive thinking with an effort and summarized display. From the 5 generation thinking cannot be switched off, so off becomes the lowest effort. Temperature is never sent from 4.7 on, nor with thinking, which also removes a 400 on the newer model presets. Thinking blocks travel to the next round unchanged, signature included; a tool round another provider started runs without thinking. Every text block is joined instead of only the first. The JSON instruction goes only to callers that ask for JSON. Unknown model names learn the dialect and the sampling rule from a refusal. --- src/lib/ai/__tests__/anthropic-client.test.ts | 7 +- .../ai/__tests__/anthropic-reasoning.test.ts | 492 +++++++++++++++++ .../anthropic-adaptive-rejected.json | 7 + .../reasoning/anthropic-after-tool.json | 17 + .../anthropic-temperature-rejected.json | 7 + .../anthropic-thinking-tool-round.json | 29 + src/lib/ai/anthropic-client.ts | 507 ++++++++++++++++-- 7 files changed, 1008 insertions(+), 58 deletions(-) create mode 100644 src/lib/ai/__tests__/anthropic-reasoning.test.ts create mode 100644 src/lib/ai/__tests__/fixtures/reasoning/anthropic-adaptive-rejected.json create mode 100644 src/lib/ai/__tests__/fixtures/reasoning/anthropic-after-tool.json create mode 100644 src/lib/ai/__tests__/fixtures/reasoning/anthropic-temperature-rejected.json create mode 100644 src/lib/ai/__tests__/fixtures/reasoning/anthropic-thinking-tool-round.json diff --git a/src/lib/ai/__tests__/anthropic-client.test.ts b/src/lib/ai/__tests__/anthropic-client.test.ts index e5761064f..a3e754b41 100644 --- a/src/lib/ai/__tests__/anthropic-client.test.ts +++ b/src/lib/ai/__tests__/anthropic-client.test.ts @@ -72,9 +72,10 @@ describe("AnthropicClient", () => { }, ]); expect(body.messages).toEqual([expect.objectContaining({ role: "user" })]); - // User message carries the original text plus the JSON-coercion instruction. - expect(body.messages[0].content).toContain("Analyze this data."); - expect(body.messages[0].content.toLowerCase()).toContain("json"); + // v1.41 — no `responseFormat: "json"`, no JSON instruction: the user + // message goes out as written. (It used to be appended to every request + // without tools, prose included.) + expect(body.messages[0].content).toBe("Analyze this data."); }); it("sends no assistant turn for JSON surfaces and instructs the last user turn", async () => { diff --git a/src/lib/ai/__tests__/anthropic-reasoning.test.ts b/src/lib/ai/__tests__/anthropic-reasoning.test.ts new file mode 100644 index 000000000..5253feb10 --- /dev/null +++ b/src/lib/ai/__tests__/anthropic-reasoning.test.ts @@ -0,0 +1,492 @@ +import { readFileSync } from "node:fs"; +import { join } from "node:path"; +import { beforeEach, describe, expect, it, vi } from "vitest"; +import { AnthropicClient } from "../anthropic-client"; +import { resetReasoningDialectCache } from "../reasoning/dialect-cache"; +import type { + AiMessage, + AiReasoningEvent, + AiToolDef, + CompletionParams, + CompletionResult, +} from "../types"; + +// safeFetch's requirePublicHost path runs through undici's own `fetch`. +// Delegate it to the global stub these tests install. +vi.mock("undici", async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + fetch: (input: unknown, init?: unknown) => + (globalThis.fetch as unknown as (i: unknown, n?: unknown) => unknown)( + input, + init, + ), + }; +}); + +/** + * v1.41 — Anthropic thinking at the wire. The response fixtures follow the + * documented Messages API shapes (thinking blocks with `signature`, + * `redacted_thinking` with `data`, interleaved text and tool_use): + * https://platform.claude.com/docs/en/build-with-claude/extended-thinking. + * No health data; signatures are placeholders. + */ +const FIXTURES = join(__dirname, "fixtures/reasoning"); +const fixture = (name: string) => readFileSync(join(FIXTURES, name), "utf8"); + +const ok = (name: string) => async () => + new Response(fixture(name), { + status: 200, + headers: { "content-type": "application/json" }, + }); +const fail = (status: number, name: string) => async () => + new Response(fixture(name), { status }); + +const TOOLS: AiToolDef[] = [ + { + name: "get_weather", + description: "Weather for a city", + parameters: { + type: "object", + properties: { city: { type: "string" } }, + required: ["city"], + }, + }, +]; + +function params(extra: Partial = {}): CompletionParams { + return { + system: "You are terse.", + messages: [{ role: "user", content: "Weather in the capital of France?" }], + maxTokens: 600, + ...extra, + }; +} + +function sent(mock: ReturnType, call = 0) { + const init = mock.mock.calls[call][1]; + return { body: JSON.parse(init.body), headers: init.headers }; +} + +const client = (model: string) => + new AnthropicClient({ apiKey: "sk-ant-test-key", model }); + +beforeEach(() => { + vi.restoreAllMocks(); + resetReasoningDialectCache(); +}); + +describe("Anthropic thinking dialects", () => { + it("keeps today's wire when no reasoning is asked, on a model that takes sampling", async () => { + const mock = vi.fn().mockImplementation(ok("anthropic-after-tool.json")); + vi.stubGlobal("fetch", mock); + await client("claude-sonnet-4-6").generateCompletion(params()); + const { body, headers } = sent(mock); + expect(body.temperature).toBe(0.3); + expect(body.max_tokens).toBe(600); + expect(body).not.toHaveProperty("thinking"); + expect(body).not.toHaveProperty("output_config"); + expect(headers).not.toHaveProperty("anthropic-beta"); + }); + + it("never sends temperature from 4.7 on, reasoning or not", async () => { + const mock = vi.fn().mockImplementation(ok("anthropic-after-tool.json")); + vi.stubGlobal("fetch", mock); + await client("claude-opus-5").generateCompletion(params()); + await client("claude-opus-4-7").generateCompletion(params()); + await client("claude-sonnet-5").generateCompletion(params()); + for (const call of [0, 1, 2]) { + expect(sent(mock, call).body).not.toHaveProperty("temperature"); + } + }); + + it("asks a 4.5 model for a thinking budget and interleaves on tool rounds", async () => { + const mock = vi + .fn() + .mockImplementation(ok("anthropic-thinking-tool-round.json")); + vi.stubGlobal("fetch", mock); + await client("claude-sonnet-4-5").generateCompletion( + params({ + tools: TOOLS, + toolChoice: "auto", + reasoning: { effort: "medium", summaries: true }, + }), + ); + const { body, headers } = sent(mock); + expect(body.thinking).toEqual({ type: "enabled", budget_tokens: 4096 }); + // budget_tokens must stay below max_tokens: answer + budget. + expect(body.max_tokens).toBe(600 + 4096); + expect(body).not.toHaveProperty("temperature"); + expect(body).not.toHaveProperty("output_config"); + expect(body.tool_choice).toEqual({ type: "auto" }); + expect(headers["anthropic-beta"]).toBe("interleaved-thinking-2025-05-14"); + }); + + it("asks 4.6 and later for adaptive thinking with an effort", async () => { + const mock = vi.fn().mockImplementation(ok("anthropic-after-tool.json")); + vi.stubGlobal("fetch", mock); + for (const model of [ + "claude-sonnet-4-6", + "claude-opus-4-7", + "claude-opus-5", + ]) { + await client(model).generateCompletion( + params({ + tools: TOOLS, + reasoning: { effort: "high", summaries: true }, + }), + ); + } + for (const call of [0, 1, 2]) { + const { body, headers } = sent(mock, call); + expect(body.thinking).toEqual({ + type: "adaptive", + display: "summarized", + }); + expect(body.output_config).toEqual({ effort: "high" }); + expect(body.max_tokens).toBe(600 + 12_000); + expect(body).not.toHaveProperty("temperature"); + expect(headers).not.toHaveProperty("anthropic-beta"); + } + }); + + it("switches off where it can and falls to low effort where thinking is always on", async () => { + const mock = vi.fn().mockImplementation(ok("anthropic-after-tool.json")); + vi.stubGlobal("fetch", mock); + const off = { reasoning: { effort: "off" as const, summaries: true } }; + await client("claude-opus-4-7").generateCompletion(params(off)); + await client("claude-sonnet-4-5").generateCompletion(params(off)); + await client("claude-opus-5").generateCompletion(params(off)); + expect(sent(mock, 0).body).not.toHaveProperty("thinking"); + expect(sent(mock, 0).body).not.toHaveProperty("temperature"); + expect(sent(mock, 1).body).not.toHaveProperty("thinking"); + expect(sent(mock, 1).body.temperature).toBe(0.3); + expect(sent(mock, 2).body.thinking).toEqual({ + type: "adaptive", + display: "summarized", + }); + expect(sent(mock, 2).body.output_config).toEqual({ effort: "low" }); + }); + + it("does not ask a model that cannot think", async () => { + const mock = vi.fn().mockImplementation(ok("anthropic-after-tool.json")); + vi.stubGlobal("fetch", mock); + await client("claude-3-5-sonnet-latest").generateCompletion( + params({ reasoning: { effort: "high", summaries: true } }), + ); + expect(sent(mock).body).not.toHaveProperty("thinking"); + expect(sent(mock).body.temperature).toBe(0.3); + }); +}); + +describe("Anthropic thinking replies", () => { + it("returns the summary after the reply and keeps the whole reply as state", async () => { + vi.stubGlobal( + "fetch", + vi.fn().mockImplementation(ok("anthropic-thinking-tool-round.json")), + ); + const events: AiReasoningEvent[] = []; + const result = await client("claude-sonnet-4-5").generateCompletion( + params({ + tools: TOOLS, + reasoning: { effort: "medium", summaries: true }, + onReasoning: (e) => events.push(e), + }), + ); + const thought = + "**Choosing the city**\nThe capital of France is Paris, so the weather lookup needs Paris."; + expect(events).toEqual([ + { kind: "title", text: "Choosing the city" }, + { kind: "text", text: thought }, + { kind: "done", text: "" }, + ]); + expect(result.reasoning).toEqual({ summary: [thought], tokens: null }); + expect(result.content).toBe("Let me look that up."); + expect(result.finishReason).toBe("tool_calls"); + expect(result.providerState).toEqual({ + providerType: "anthropic", + model: "claude-sonnet-4-5", + items: JSON.parse(fixture("anthropic-thinking-tool-round.json")).content, + }); + }); + + it("joins every text block, not just the first", async () => { + vi.stubGlobal( + "fetch", + vi.fn().mockImplementation(ok("anthropic-after-tool.json")), + ); + const result = await client("claude-sonnet-4-5").generateCompletion( + params({ reasoning: { effort: "low", summaries: true } }), + ); + expect(result.content).toBe("Paris is at 18 °C and cloudy."); + // A redacted block has no readable text, but it is state. + expect(result.reasoning?.summary).toEqual([]); + expect(result.providerState?.items[0]).toEqual({ + type: "redacted_thinking", + data: "redacted-fixture-b1", + }); + }); + + it("no longer tells a prose request to answer in JSON", async () => { + const mock = vi.fn().mockImplementation(ok("anthropic-after-tool.json")); + vi.stubGlobal("fetch", mock); + await client("claude-sonnet-4-6").generateCompletion(params()); + await client("claude-sonnet-4-6").generateCompletion( + params({ responseFormat: "json" }), + ); + expect(sent(mock, 0).body.messages[0].content).toBe( + "Weather in the capital of France?", + ); + expect(sent(mock, 1).body.messages[0].content).toMatch( + /single JSON object/, + ); + }); +}); + +describe("Anthropic thinking round trip", () => { + async function firstRound( + model = "claude-sonnet-4-5", + ): Promise { + vi.stubGlobal( + "fetch", + vi.fn().mockImplementation(ok("anthropic-thinking-tool-round.json")), + ); + return client(model).generateCompletion( + params({ + tools: TOOLS, + reasoning: { effort: "medium", summaries: true }, + }), + ); + } + + function next( + first: CompletionResult, + state = first.providerState, + ): AiMessage[] { + return [ + { role: "user", content: "Weather in the capital of France?" }, + { + role: "assistant", + content: first.content, + toolCalls: first.toolCalls, + providerState: state, + }, + { + role: "tool", + toolCallId: first.toolCalls![0].id, + content: '{"tempC":18}', + }, + ]; + } + + it("hands the assistant reply back unchanged, signature included", async () => { + const first = await firstRound(); + const mock = vi.fn().mockImplementation(ok("anthropic-after-tool.json")); + vi.stubGlobal("fetch", mock); + await client("claude-sonnet-4-5").generateCompletion( + params({ + messages: next(first), + tools: TOOLS, + reasoning: { effort: "medium", summaries: true }, + }), + ); + const { body } = sent(mock); + expect(body.messages[1]).toEqual({ + role: "assistant", + content: JSON.parse(fixture("anthropic-thinking-tool-round.json")) + .content, + }); + expect(body.thinking).toEqual({ type: "enabled", budget_tokens: 4096 }); + }); + + it("leads with the thinking blocks when the turn's calls changed", async () => { + const first = await firstRound(); + const mock = vi.fn().mockImplementation(ok("anthropic-after-tool.json")); + vi.stubGlobal("fetch", mock); + const messages = next(first); + messages[1] = { + ...messages[1], + toolCalls: [{ id: "toolu_other", name: "get_weather", arguments: "{}" }], + }; + messages[2] = { ...messages[2], toolCallId: "toolu_other" }; + await client("claude-sonnet-4-5").generateCompletion( + params({ + messages, + tools: TOOLS, + reasoning: { effort: "medium", summaries: true }, + }), + ); + const content = sent(mock).body.messages[1].content; + expect(content.map((b: { type: string }) => b.type)).toEqual([ + "thinking", + "text", + "tool_use", + ]); + expect(content[0].signature).toBe("sig-fixture-a1"); + expect(content[2].id).toBe("toolu_other"); + }); + + it("turns thinking off for a tool round another provider started", async () => { + const first = await firstRound(); + const mock = vi.fn().mockImplementation(ok("anthropic-after-tool.json")); + vi.stubGlobal("fetch", mock); + const fromCodex = { + providerType: "codex" as const, + model: "gpt-5.5", + items: [{ type: "reasoning" }], + }; + const result = await client("claude-sonnet-4-5").generateCompletion( + params({ + messages: next(first, fromCodex), + tools: TOOLS, + reasoning: { effort: "medium", summaries: true }, + }), + ); + const { body, headers } = sent(mock); + expect(body).not.toHaveProperty("thinking"); + expect(headers).not.toHaveProperty("anthropic-beta"); + // The foreign state never reaches the wire. + expect(JSON.stringify(body)).not.toContain('"reasoning"'); + expect( + body.messages[1].content.map((b: { type: string }) => b.type), + ).toEqual(["text", "tool_use"]); + // Not a refusal by the provider: no downgrade reported. + expect(result.reasoning?.downgradedTo).toBeUndefined(); + }); +}); + +describe("Anthropic thinking refusals", () => { + it("falls from adaptive to a budget on an unknown name, and remembers", async () => { + const mock = vi + .fn() + .mockImplementationOnce(fail(400, "anthropic-adaptive-rejected.json")) + .mockImplementation(ok("anthropic-after-tool.json")); + vi.stubGlobal("fetch", mock); + const c = client("my-claude-proxy"); + const result = await c.generateCompletion( + params({ reasoning: { effort: "low", summaries: true } }), + ); + expect(sent(mock, 0).body.thinking.type).toBe("adaptive"); + expect(sent(mock, 1).body.thinking).toEqual({ + type: "enabled", + budget_tokens: 1024, + }); + expect(sent(mock, 1).body).not.toHaveProperty("output_config"); + expect(result.content).toBe("Paris is at 18 °C and cloudy."); + expect(result.reasoning?.downgradedTo).toBeUndefined(); + + await c.generateCompletion( + params({ reasoning: { effort: "low", summaries: true } }), + ); + expect(mock).toHaveBeenCalledTimes(3); + expect(sent(mock, 2).body.thinking.type).toBe("enabled"); + }); + + it("answers without thinking when every form is refused", async () => { + const mock = vi + .fn() + .mockImplementationOnce(fail(400, "anthropic-adaptive-rejected.json")) + .mockImplementationOnce(fail(400, "anthropic-adaptive-rejected.json")) + .mockImplementation(ok("anthropic-after-tool.json")); + vi.stubGlobal("fetch", mock); + const result = await client("my-claude-proxy").generateCompletion( + params({ reasoning: { effort: "medium", summaries: true } }), + ); + expect(mock).toHaveBeenCalledTimes(3); + expect(sent(mock, 2).body).not.toHaveProperty("thinking"); + expect(result.reasoning?.downgradedTo).toBe("off"); + }); + + it("drops temperature an unknown model refuses, and remembers", async () => { + const mock = vi + .fn() + .mockImplementationOnce(fail(400, "anthropic-temperature-rejected.json")) + .mockImplementation(ok("anthropic-after-tool.json")); + vi.stubGlobal("fetch", mock); + const c = client("my-claude-proxy"); + await c.generateCompletion(params()); + expect(sent(mock, 0).body.temperature).toBe(0.3); + expect(sent(mock, 1).body).not.toHaveProperty("temperature"); + await c.generateCompletion(params()); + expect(mock).toHaveBeenCalledTimes(3); + expect(sent(mock, 2).body).not.toHaveProperty("temperature"); + }); + + it("raises a 400 that is about something else, once", async () => { + const mock = vi.fn().mockImplementation( + async () => + new Response( + JSON.stringify({ + type: "error", + error: { + type: "invalid_request_error", + message: "messages: too long", + }, + }), + { status: 400 }, + ), + ); + vi.stubGlobal("fetch", mock); + await expect( + client("claude-opus-5").generateCompletion( + params({ reasoning: { effort: "high", summaries: true } }), + ), + ).rejects.toMatchObject({ httpStatus: 400, upstream: "anthropic" }); + expect(mock).toHaveBeenCalledTimes(1); + }); +}); + +describe("Anthropic adaptive rounds that chose not to think", () => { + it("keeps thinking on for the next round of its own turn", async () => { + const noThought = { + id: "msg_fixture_03", + type: "message", + role: "assistant", + model: "claude-opus-5", + content: [ + { + type: "tool_use", + id: "toolu_fixture_02", + name: "get_weather", + input: { city: "Paris" }, + }, + ], + stop_reason: "tool_use", + usage: { input_tokens: 300, output_tokens: 20 }, + }; + const mock = vi + .fn() + .mockImplementationOnce( + async () => new Response(JSON.stringify(noThought), { status: 200 }), + ) + .mockImplementation(ok("anthropic-after-tool.json")); + vi.stubGlobal("fetch", mock); + const c = client("claude-opus-5"); + const reasoning = { effort: "medium" as const, summaries: true }; + const first = await c.generateCompletion( + params({ tools: TOOLS, reasoning }), + ); + expect(first.providerState?.items).toEqual(noThought.content); + await c.generateCompletion( + params({ + tools: TOOLS, + reasoning, + messages: [ + { role: "user", content: "Weather?" }, + { + role: "assistant", + content: first.content, + toolCalls: first.toolCalls, + providerState: first.providerState, + }, + { role: "tool", toolCallId: "toolu_fixture_02", content: "{}" }, + ], + }), + ); + expect(sent(mock, 1).body.thinking).toEqual({ + type: "adaptive", + display: "summarized", + }); + }); +}); diff --git a/src/lib/ai/__tests__/fixtures/reasoning/anthropic-adaptive-rejected.json b/src/lib/ai/__tests__/fixtures/reasoning/anthropic-adaptive-rejected.json new file mode 100644 index 000000000..3f028eb73 --- /dev/null +++ b/src/lib/ai/__tests__/fixtures/reasoning/anthropic-adaptive-rejected.json @@ -0,0 +1,7 @@ +{ + "type": "error", + "error": { + "type": "invalid_request_error", + "message": "thinking.type: Input tag 'adaptive' found using 'type' does not match any of the expected tags: 'disabled', 'enabled'" + } +} diff --git a/src/lib/ai/__tests__/fixtures/reasoning/anthropic-after-tool.json b/src/lib/ai/__tests__/fixtures/reasoning/anthropic-after-tool.json new file mode 100644 index 000000000..325cd97dc --- /dev/null +++ b/src/lib/ai/__tests__/fixtures/reasoning/anthropic-after-tool.json @@ -0,0 +1,17 @@ +{ + "id": "msg_fixture_02", + "type": "message", + "role": "assistant", + "model": "claude-sonnet-4-5", + "content": [ + { "type": "redacted_thinking", "data": "redacted-fixture-b1" }, + { "type": "text", "text": "Paris is at 18 °C" }, + { "type": "text", "text": " and cloudy." } + ], + "stop_reason": "end_turn", + "usage": { + "input_tokens": 760, + "output_tokens": 41, + "cache_read_input_tokens": 512 + } +} diff --git a/src/lib/ai/__tests__/fixtures/reasoning/anthropic-temperature-rejected.json b/src/lib/ai/__tests__/fixtures/reasoning/anthropic-temperature-rejected.json new file mode 100644 index 000000000..3d93c3cb0 --- /dev/null +++ b/src/lib/ai/__tests__/fixtures/reasoning/anthropic-temperature-rejected.json @@ -0,0 +1,7 @@ +{ + "type": "error", + "error": { + "type": "invalid_request_error", + "message": "temperature: this model does not accept sampling parameters" + } +} diff --git a/src/lib/ai/__tests__/fixtures/reasoning/anthropic-thinking-tool-round.json b/src/lib/ai/__tests__/fixtures/reasoning/anthropic-thinking-tool-round.json new file mode 100644 index 000000000..a76ad54d0 --- /dev/null +++ b/src/lib/ai/__tests__/fixtures/reasoning/anthropic-thinking-tool-round.json @@ -0,0 +1,29 @@ +{ + "id": "msg_fixture_01", + "type": "message", + "role": "assistant", + "model": "claude-sonnet-4-5", + "content": [ + { + "type": "thinking", + "thinking": "**Choosing the city**\nThe capital of France is Paris, so the weather lookup needs Paris.", + "signature": "sig-fixture-a1" + }, + { + "type": "text", + "text": "Let me look that up." + }, + { + "type": "tool_use", + "id": "toolu_fixture_01", + "name": "get_weather", + "input": { "city": "Paris" } + } + ], + "stop_reason": "tool_use", + "usage": { + "input_tokens": 612, + "output_tokens": 118, + "cache_read_input_tokens": 0 + } +} diff --git a/src/lib/ai/anthropic-client.ts b/src/lib/ai/anthropic-client.ts index 6f4b460fe..558985f2a 100644 --- a/src/lib/ai/anthropic-client.ts +++ b/src/lib/ai/anthropic-client.ts @@ -5,11 +5,29 @@ import type { AIProvider, AiContentPart, AiMessage, + AiProviderState, AiToolCall, AiToolDef, CompletionParams, CompletionResult, } from "./types"; +import { + annotateReasoningDowngrade, + learnedReasoningDialect, + rememberReasoningDialect, +} from "./reasoning/dialect-cache"; +import { + REASONING_LEVELS, + REASONING_THINKING_BUDGET, + type ReasoningLevel, +} from "./reasoning/levels"; +import { + anthropicRefusesSampling, + anthropicThinkingDialect, + anthropicThinksAlways, + reasoningTitleOf, + type AnthropicThinkingDialect, +} from "./reasoning/support"; /** * v1.18.9 — Anthropic Messages API content blocks. The text-only path sends a @@ -42,13 +60,37 @@ type AnthropicContentBlock = type: "tool_result"; tool_use_id: string; content: string; - }; + } + // v1.41 — `thinking` / `redacted_thinking` blocks of an earlier round, + // replayed byte-for-byte from `providerState` (signature included). + | Record; interface AnthropicWireMessage { role: "user" | "assistant"; content: string | AnthropicContentBlock[]; } +/** + * The thinking blocks a model returned, in the shape the next round must hand + * back: "you must pass thinking blocks back to the API … the complete, + * unmodified block", with `signature` (`thinking`) or `data` + * (`redacted_thinking`). https://platform.claude.com/docs/en/build-with-claude/extended-thinking + */ +function isThinkingBlock(block: unknown): block is Record { + const type = (block as { type?: unknown } | null)?.type; + return type === "thinking" || type === "redacted_thinking"; +} + +/** `state` when this client made it for `model`, else `null`. */ +function ownState( + state: AiProviderState | undefined, + model: string, +): AiProviderState | null { + return state && state.providerType === "anthropic" && state.model === model + ? state + : null; +} + interface AnthropicClientConfig { apiKey: string; model: string; @@ -58,6 +100,27 @@ interface AnthropicClientConfig { const DEFAULT_BASE_URL = "https://api.anthropic.com/v1"; const ANTHROPIC_VERSION = "2023-06-01"; +/** + * Manual (budget) thinking keeps its thoughts between tool calls only with + * this beta header; adaptive thinking interleaves on its own. + * https://platform.claude.com/docs/en/build-with-claude/extended-thinking#interleaved-thinking + */ +const INTERLEAVED_THINKING_BETA = "interleaved-thinking-2025-05-14"; + +/** What one request asks for. */ +type ThinkingPlan = + | { kind: "none" } + | { kind: "manual"; budget: number; level: Exclude } + | { kind: "adaptive"; effort: Exclude }; + +/** What a model behind an endpoint was learned to refuse. */ +interface AnthropicDialect { + /** The thinking form that worked after the name-derived one was refused. */ + thinking?: AnthropicThinkingDialect; + /** The model refused `temperature`. */ + noSampling?: true; +} + // The only JSON steering on this wire. Current Anthropic models reject an // assistant-turn prefill (the old `{` trick) with HTTP 400, so the contract // rides on the instruction and on `extractJsonObject` on the way out. @@ -104,7 +167,7 @@ function mapParts(parts: AiContentPart[]): AnthropicContentBlock[] { * a `role:"user"` turn carrying a `tool_result` block (Anthropic's convention). * An assistant turn with `toolCalls` emits `tool_use` blocks alongside any text. */ -function mapMessage(m: AiMessage): AnthropicWireMessage { +function mapMessage(m: AiMessage, model: string): AnthropicWireMessage { if (m.role === "tool") { return { role: "user", @@ -120,6 +183,41 @@ function mapMessage(m: AiMessage): AnthropicWireMessage { const hasToolCalls = m.role === "assistant" && !!m.toolCalls && m.toolCalls.length > 0; + // v1.41 — an assistant round that thought. When the stored reply's tool + // calls are exactly the ones this turn carries, the reply goes back as it + // came (interleaved thinking keeps its order); otherwise its thinking + // blocks lead and the text and tool_use blocks are rebuilt behind them. + const state = + m.role === "assistant" ? ownState(m.providerState, model) : null; + if (state) { + const stored = state.items.filter( + (b): b is Record => b !== null && typeof b === "object", + ); + const storedIds = stored + .filter((b) => b.type === "tool_use") + .map((b) => String(b.id)) + .sort(); + const turnIds = (m.toolCalls ?? []).map((tc) => tc.id).sort(); + if ( + storedIds.length > 0 && + storedIds.length === turnIds.length && + storedIds.every((id, i) => id === turnIds[i]) + ) { + return { role: "assistant", content: stored }; + } + const thinking = stored.filter(isThinkingBlock); + if (thinking.length > 0) { + const rebuilt = mapMessage({ ...m, providerState: undefined }, model); + const rest = + typeof rebuilt.content === "string" + ? rebuilt.content.length > 0 + ? [{ type: "text" as const, text: rebuilt.content }] + : [] + : rebuilt.content; + return { role: "assistant", content: [...thinking, ...rest] }; + } + } + // Plain-string content with no tool calls stays a bare string on the wire — // byte-identical to the pre-refactor text-only path. Only build a block array // when there are content parts or tool_use blocks to carry. @@ -180,15 +278,17 @@ export class AnthropicClient implements AIProvider { // JSON body; the object extraction below stays off that path. const wantsJson = params.responseFormat === "json" && !hasTools; - const wireMessages: AnthropicWireMessage[] = - params.messages.map(mapMessage); + const wireMessages: AnthropicWireMessage[] = params.messages.map((m) => + mapMessage(m, this.config.model), + ); - // JSON-reliability instruction. The pre-refactor client appended this to - // the user prompt UNCONDITIONALLY (not gated on `responseFormat`), so every - // Anthropic caller — including the Coach prose path — saw it. Preserve that - // exactly by appending to the LAST user turn's text. Tools imply the - // non-JSON tool flow, so skip the instruction when tools are present. - if (!hasTools) { + // JSON-reliability instruction, only for callers that asked for JSON. + // v1.41 — it used to ride on every request without tools, so the Coach's + // prose rounds, the judge's final-line verdict and the array-shaped memory + // extractors were all told to "reply with a single JSON object". Every + // object-shaped caller sets `responseFormat: "json"` (the status provider + // does by default), so gating on it changes nothing for them. + if (wantsJson) { for (let i = wireMessages.length - 1; i >= 0; i -= 1) { if (wireMessages[i].role !== "user") continue; const turn = wireMessages[i]; @@ -199,7 +299,7 @@ export class AnthropicClient implements AIProvider { // end with the instruction text) so the wire stays a single text // block — byte-identical to the pre-refactor `wrapForJson` output. const last = turn.content[turn.content.length - 1]; - if (last && last.type === "text") { + if (last && last.type === "text" && typeof last.text === "string") { last.text = `${last.text}${JSON_INSTRUCTION}`; } else { turn.content = [ @@ -228,6 +328,7 @@ export class AnthropicClient implements AIProvider { const tools = hasTools ? buildAnthropicTools(params.tools as AiToolDef[]) : undefined; + // Thinking allows only `auto` and `none`, which is all this maps to. const toolChoice = hasTools && params.toolChoice ? params.toolChoice === "none" @@ -235,40 +336,48 @@ export class AnthropicClient implements AIProvider { : { type: "auto" as const } : undefined; - const res = await safeFetch( - url, - { - method: "POST", - headers: { - "Content-Type": "application/json", - "x-api-key": this.config.apiKey, - "anthropic-version": ANTHROPIC_VERSION, - }, - // NOTE: Anthropic's Messages API has no `seed` parameter, so - // `params.seed` is intentionally not forwarded here — output on this - // provider is non-deterministic regardless of the pinned seed. - body: JSON.stringify({ - model: this.config.model, - max_tokens: params.maxTokens ?? 1000, - temperature: params.temperature ?? 0.3, - system, - messages: wireMessages, - ...(tools ? { tools } : {}), - ...(toolChoice ? { tool_choice: toolChoice } : {}), - }), - }, - // 60 s ceiling — see openai-client.ts for the rationale. - // v1.11.2 — base URL is user/admin-overridable; pin the connect-time DNS - // check so a private/metadata address is rejected (SSRF/rebinding). - // v1.20.1 — compose the caller's cancel signal (Coach SSE disconnect) so - // a mid-generation abort tears the upstream call down early. - // The record owner's response-timeout setting, else the surface value, else 60 s. - { - timeoutMs: callTimeoutMs(params, this.responseTimeoutSeconds), - requirePublicHost: true, - signal: params.signal, - }, - ); + const dialect = this.readDialect(); + let plan = this.planThinking(params, dialect); + let sendTemperature = this.sendsTemperature(plan, dialect); + + let res = await this.send(url, params, { + plan, + sendTemperature, + system, + wireMessages, + tools, + toolChoice, + hasTools, + }); + + // v1.41 — a model that refuses the thinking form or the sampling field it + // was sent is asked again without it, and the endpoint remembers. Each + // step drops something, so the loop ends; the answer is never lost to a + // reasoning parameter. + let rawBody = ""; + for (let step = 0; !res.ok && step < 3; step += 1) { + rawBody = await res.text().catch(() => ""); + if (res.status !== 400) break; + const learned = this.learnFromRejection( + plan, + sendTemperature, + rawBody, + dialect, + ); + if (!learned) break; + plan = this.planThinking(params, dialect); + sendTemperature = this.sendsTemperature(plan, dialect); + rawBody = ""; + res = await this.send(url, params, { + plan, + sendTemperature, + system, + wireMessages, + tools, + toolChoice, + hasTools, + }); + } if (!res.ok) { // Mirror the openai-client body-capture so 4xx/5xx upstream incidents @@ -276,11 +385,8 @@ export class AnthropicClient implements AIProvider { // instead of surfacing as an opaque "Anthropic request failed (5xx)". // Strip anything that looks like an Anthropic API key from the excerpt // before logging. - const rawBody = await res.text().catch(() => ""); - const bodyExcerpt = rawBody - .slice(0, 500) - .replace(/sk-ant-[A-Za-z0-9_-]{8,}/g, "sk-ant-***redacted***") - .replace(/Bearer\s+[A-Za-z0-9_.-]+/gi, "Bearer ***redacted***"); + if (!rawBody) rawBody = await res.text().catch(() => ""); + const bodyExcerpt = redactAnthropicBody(rawBody); const err = new Error(`Anthropic request failed (${res.status})`); Object.assign(err, { httpStatus: res.status, @@ -295,6 +401,7 @@ export class AnthropicClient implements AIProvider { content?: Array<{ type: string; text?: string; + thinking?: string; id?: string; name?: string; input?: Record; @@ -304,16 +411,34 @@ export class AnthropicClient implements AIProvider { input_tokens?: number; output_tokens?: number; cache_read_input_tokens?: number; + output_tokens_details?: { thinking_tokens?: number }; }; }; - const textBlock = json.content?.find((c) => c.type === "text"); - const rawText = textBlock?.text; + const blocks = json.content ?? []; + + // v1.41 — every text block, not just the first: with thinking on, a reply + // can carry text before and after its thinking or tool_use blocks. Text + // blocks that follow each other directly are one passage split by the + // API; a non-text block between them is a paragraph break. + let rawText: string | undefined; + let brokenSinceText = false; + for (const block of blocks) { + if (block.type !== "text") { + brokenSinceText = true; + continue; + } + const text = block.text ?? ""; + rawText = + rawText === undefined + ? text + : `${rawText}${brokenSinceText && text ? "\n\n" : ""}${text}`; + brokenSinceText = false; + } + if (rawText !== undefined && rawText.trim() === "") rawText = undefined; // v1.20.0 — surface tool_use blocks for F1's loop. - const toolUseBlocks = (json.content ?? []).filter( - (c) => c.type === "tool_use", - ); + const toolUseBlocks = blocks.filter((c) => c.type === "tool_use"); const toolCalls: AiToolCall[] | undefined = toolUseBlocks.length > 0 ? toolUseBlocks.map((b) => ({ @@ -338,6 +463,23 @@ export class AnthropicClient implements AIProvider { throw err; } + // v1.41 — thinking. The blocks travel to the next round untouched (the + // whole reply, so interleaved order survives); readable summaries go to + // the receiver after the reply, since this client does not stream. + const thinkingBlocks = blocks.filter(isThinkingBlock); + const summary = thinkingBlocks + .map((b) => (typeof b.thinking === "string" ? b.thinking.trim() : "")) + .filter((t) => t.length > 0); + const emit = params.reasoning ? params.onReasoning : undefined; + if (emit) { + for (const text of summary) { + const title = reasoningTitleOf(text); + if (title) emit({ kind: "title", text: title }); + emit({ kind: "text", text }); + } + emit({ kind: "done", text: "" }); + } + // Without a prefill the model returns the whole object, sometimes inside // a ```json fence or behind a short lead-in sentence; narrow to the // object so the caller parses it as-is. @@ -366,6 +508,22 @@ export class AnthropicClient implements AIProvider { ? "stop" : undefined; + const sentLevel = levelOfPlan(plan); + const reasoning: CompletionResult["reasoning"] = params.reasoning + ? { + summary, + // Thinking is billed as output and already inside `output_tokens`. + tokens: json.usage?.output_tokens_details?.thinking_tokens ?? null, + // Only a refusal the endpoint taught counts as a downgrade; a turn + // that started elsewhere and so runs without thinking does not. + ...(dialect.thinking !== undefined && + REASONING_LEVELS.indexOf(sentLevel) < + REASONING_LEVELS.indexOf(params.reasoning.effort) + ? { downgradedTo: sentLevel } + : {}), + } + : undefined; + return { content, tokensUsed, @@ -374,6 +532,245 @@ export class AnthropicClient implements AIProvider { providerType: "anthropic", ...(toolCalls ? { toolCalls } : {}), finishReason, + ...(reasoning ? { reasoning } : {}), + // State goes back whenever the reply thought, and on every tool round + // of a call that asked for reasoning: an adaptive round may choose not + // to think, and the next round must still recognise it as its own. + ...(thinkingBlocks.length > 0 || (params.reasoning && toolCalls) + ? { + providerState: { + providerType: "anthropic" as const, + model: this.config.model, + items: blocks, + }, + } + : {}), }; } + + // ── v1.41 reasoning ─────────────────────────────────────────────────────── + + private dialectKey() { + return { + provider: "anthropic", + endpoint: this.config.baseUrl ?? DEFAULT_BASE_URL, + model: this.config.model, + }; + } + + private readDialect(): AnthropicDialect { + const raw = learnedReasoningDialect(this.dialectKey()); + if (!raw) return {}; + try { + return JSON.parse(raw) as AnthropicDialect; + } catch { + return {}; + } + } + + /** + * The thinking this request asks for. Absent `params.reasoning` → none, + * today's wire. The dialect comes from the model name (manual budget up to + * 4.5, adaptive effort from 4.6) unless the endpoint taught otherwise. + * + * A turn that began on another provider (or another model) carries + * assistant tool rounds without our thinking blocks; with thinking on, the + * API expects the current tool round to start with one, so thinking stays + * off for the rest of that turn. + */ + private planThinking( + params: CompletionParams, + dialect: AnthropicDialect, + ): ThinkingPlan { + const reasoning = params.reasoning; + if (!reasoning) return { kind: "none" }; + const form = + dialect.thinking ?? anthropicThinkingDialect(this.config.model); + if (form === "none") return { kind: "none" }; + if (this.toolRoundStartedElsewhere(params.messages, form)) { + return { kind: "none" }; + } + const level = reasoning.effort; + if (form === "manual") { + return level === "off" + ? { kind: "none" } + : { kind: "manual", budget: REASONING_THINKING_BUDGET[level], level }; + } + if (level === "off") { + // From the 5 generation thinking cannot be switched off; the lowest + // effort is the honest "off" (`offIsReal: false` in support.ts). + return anthropicThinksAlways(this.config.model) + ? { kind: "adaptive", effort: "low" } + : { kind: "none" }; + } + return { kind: "adaptive", effort: level }; + } + + /** + * True when the current tool round (the turns after the last user message) + * has an assistant turn this client did not make for this model: another + * provider, or another model, started it. In manual mode a turn of ours + * that ran without thinking counts too, because a budget-mode continuation + * must open with a thinking block. In adaptive mode the model may have + * chosen not to think in that round, which the API accepts. + */ + private toolRoundStartedElsewhere( + messages: AiMessage[], + form: AnthropicThinkingDialect, + ): boolean { + let lastUser = -1; + messages.forEach((m, i) => { + if (m.role === "user") lastUser = i; + }); + return messages.slice(lastUser + 1).some((m) => { + if (m.role !== "assistant" || (m.toolCalls?.length ?? 0) === 0) { + return false; + } + const own = ownState(m.providerState, this.config.model); + if (!own) return true; + return form === "manual" && !own.items.some(isThinkingBlock); + }); + } + + /** + * `temperature` goes out only where the model takes it: never from 4.7 on + * (the 5 generation refuses even the default), never with thinking on, and + * never once the endpoint refused it. + */ + private sendsTemperature( + plan: ThinkingPlan, + dialect: AnthropicDialect, + ): boolean { + return ( + plan.kind === "none" && + !dialect.noSampling && + !anthropicRefusesSampling(this.config.model) + ); + } + + /** + * Learn from a 400 that names a parameter we sent. Returns false when the + * body is about something else; the caller then raises it as before. + */ + private learnFromRejection( + plan: ThinkingPlan, + sentTemperature: boolean, + body: string, + dialect: AnthropicDialect, + ): boolean { + const lower = body.toLowerCase(); + let changed: string | null = null; + if (sentTemperature && /temperature|top_p|sampling/.test(lower)) { + dialect.noSampling = true; + changed = "sampling"; + } else if ( + plan.kind !== "none" && + /thinking|budget_tokens|adaptive|output_config|effort/.test(lower) + ) { + dialect.thinking = plan.kind === "adaptive" ? "manual" : "none"; + changed = plan.kind; + } + if (!changed) return false; + rememberReasoningDialect(this.dialectKey(), JSON.stringify(dialect)); + annotateReasoningDowngrade( + "anthropic", + changed, + changed === "sampling" ? "no-sampling" : (dialect.thinking ?? "none"), + ); + return true; + } + + private send( + url: string, + params: CompletionParams, + req: { + plan: ThinkingPlan; + sendTemperature: boolean; + system: unknown; + wireMessages: AnthropicWireMessage[]; + tools: unknown; + toolChoice: unknown; + hasTools: boolean; + }, + ): Promise { + const { plan } = req; + const answerBudget = params.maxTokens ?? 1000; + // `max_tokens` bounds thinking and answer together, so the thinking + // budget rides on top of the answer budget; manual mode additionally + // requires `budget_tokens < max_tokens`. It is a ceiling, not a charge. + const maxTokens = + plan.kind === "manual" + ? answerBudget + plan.budget + : plan.kind === "adaptive" + ? answerBudget + REASONING_THINKING_BUDGET[plan.effort] + : answerBudget; + const headers: Record = { + "Content-Type": "application/json", + "x-api-key": this.config.apiKey, + "anthropic-version": ANTHROPIC_VERSION, + }; + if (plan.kind === "manual" && req.hasTools) { + headers["anthropic-beta"] = INTERLEAVED_THINKING_BETA; + } + return safeFetch( + url, + { + method: "POST", + headers, + // NOTE: Anthropic's Messages API has no `seed` parameter, so + // `params.seed` is intentionally not forwarded here — output on this + // provider is non-deterministic regardless of the pinned seed. + body: JSON.stringify({ + model: this.config.model, + max_tokens: maxTokens, + ...(req.sendTemperature + ? { temperature: params.temperature ?? 0.3 } + : {}), + // Manual: a fixed budget. Adaptive: the model decides how much, + // `effort` steers it, and `display: "summarized"` asks for readable + // thinking where the default would leave it empty (5 generation). + // https://platform.claude.com/docs/en/build-with-claude/adaptive-thinking + ...(plan.kind === "manual" + ? { thinking: { type: "enabled", budget_tokens: plan.budget } } + : plan.kind === "adaptive" + ? { + thinking: { type: "adaptive", display: "summarized" }, + output_config: { effort: plan.effort }, + } + : {}), + system: req.system, + messages: req.wireMessages, + ...(req.tools ? { tools: req.tools } : {}), + ...(req.toolChoice ? { tool_choice: req.toolChoice } : {}), + }), + }, + // 60 s ceiling — see openai-client.ts for the rationale. + // v1.11.2 — base URL is user/admin-overridable; pin the connect-time DNS + // check so a private/metadata address is rejected (SSRF/rebinding). + // v1.20.1 — compose the caller's cancel signal (Coach SSE disconnect) so + // a mid-generation abort tears the upstream call down early. + // The record owner's response-timeout setting, else the surface value, else 60 s. + { + timeoutMs: callTimeoutMs(params, this.responseTimeoutSeconds), + requirePublicHost: true, + signal: params.signal, + }, + ); + } +} + +/** The level a plan delivers, for `downgradedTo`. */ +function levelOfPlan(plan: ThinkingPlan): ReasoningLevel { + return plan.kind === "manual" + ? plan.level + : plan.kind === "adaptive" + ? plan.effort + : "off"; +} + +function redactAnthropicBody(rawBody: string): string { + return rawBody + .slice(0, 500) + .replace(/sk-ant-[A-Za-z0-9_-]{8,}/g, "sk-ant-***redacted***") + .replace(/Bearer\s+[A-Za-z0-9_.-]+/gi, "Bearer ***redacted***"); } From 02818832472c619261645fbddedf96b4a4aa831e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Tue, 6 Oct 2026 19:26:29 +0200 Subject: [PATCH 4/7] feat(ai): send reasoning on chat-completions endpoints and local servers OpenAI sends reasoning_effort for reasoning models on rounds without tools, off mapped to each family's lowest value. Gateways send reasoning_effort, OpenRouter its reasoning object, and the reasoning details or thinking blocks a gateway returns go back on the next round's assistant message. Local servers send reasoning_effort and read reasoning_content, reasoning and a leading think block, live on the stream without letting it reach the answer. The thinking budget rides on top of the answer budget where reasoning counts against it. A refused reasoning field is retried once without it and remembered per endpoint and model, before the JSON dialect check. A call's own level wins over the stored chain setting, which still applies when the call has none. --- .../reasoning/gateway-reasoning-rejected.json | 8 + .../openrouter-claude-after-tool.json | 36 +++ .../openrouter-claude-tool-round.json | 56 ++++ src/lib/ai/__tests__/local-reasoning.test.ts | 278 +++++++++++++++++ src/lib/ai/__tests__/openai-reasoning.test.ts | 287 ++++++++++++++++++ src/lib/ai/local-client.ts | 239 ++++++++++++++- src/lib/ai/openai-client.ts | 216 ++++++++++++- 7 files changed, 1104 insertions(+), 16 deletions(-) create mode 100644 src/lib/ai/__tests__/fixtures/reasoning/gateway-reasoning-rejected.json create mode 100644 src/lib/ai/__tests__/fixtures/reasoning/openrouter-claude-after-tool.json create mode 100644 src/lib/ai/__tests__/fixtures/reasoning/openrouter-claude-tool-round.json create mode 100644 src/lib/ai/__tests__/local-reasoning.test.ts create mode 100644 src/lib/ai/__tests__/openai-reasoning.test.ts diff --git a/src/lib/ai/__tests__/fixtures/reasoning/gateway-reasoning-rejected.json b/src/lib/ai/__tests__/fixtures/reasoning/gateway-reasoning-rejected.json new file mode 100644 index 000000000..1157ee15d --- /dev/null +++ b/src/lib/ai/__tests__/fixtures/reasoning/gateway-reasoning-rejected.json @@ -0,0 +1,8 @@ +{ + "error": { + "message": "Unrecognized request argument supplied: reasoning_effort", + "type": "invalid_request_error", + "param": null, + "code": null + } +} diff --git a/src/lib/ai/__tests__/fixtures/reasoning/openrouter-claude-after-tool.json b/src/lib/ai/__tests__/fixtures/reasoning/openrouter-claude-after-tool.json new file mode 100644 index 000000000..12698c190 --- /dev/null +++ b/src/lib/ai/__tests__/fixtures/reasoning/openrouter-claude-after-tool.json @@ -0,0 +1,36 @@ +{ + "id": "gen-fixture", + "object": "chat.completion", + "model": "anthropic/claude-haiku-4.5", + "service_tier": "default", + "choices": [ + { + "index": 0, + "logprobs": null, + "finish_reason": "stop", + "native_finish_reason": "end_turn", + "message": { + "role": "assistant", + "content": "Paris (the capital of France) currently has cloudy skies and a temperature of 18°C.", + "refusal": null, + "reasoning": null + } + } + ], + "usage": { + "prompt_tokens": 746, + "completion_tokens": 25, + "total_tokens": 771, + "prompt_tokens_details": { + "cached_tokens": 0, + "cache_write_tokens": 0, + "audio_tokens": 0, + "video_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0, + "image_tokens": 0, + "audio_tokens": 0 + } + } +} diff --git a/src/lib/ai/__tests__/fixtures/reasoning/openrouter-claude-tool-round.json b/src/lib/ai/__tests__/fixtures/reasoning/openrouter-claude-tool-round.json new file mode 100644 index 000000000..2ea53b8d8 --- /dev/null +++ b/src/lib/ai/__tests__/fixtures/reasoning/openrouter-claude-tool-round.json @@ -0,0 +1,56 @@ +{ + "id": "gen-fixture", + "object": "chat.completion", + "model": "anthropic/claude-haiku-4.5", + "service_tier": "default", + "choices": [ + { + "index": 0, + "logprobs": null, + "finish_reason": "tool_calls", + "native_finish_reason": "tool_use", + "message": { + "role": "assistant", + "content": null, + "refusal": null, + "reasoning": "The user is asking me to:\n1. Think about which city is the capital of France\n2. Call get_weather for that city\n\nThe capital of France is Paris. So I should call get_weather with \"Paris\" as the city parameter.", + "tool_calls": [ + { + "type": "function", + "index": 0, + "id": "toolu_bdrk_014bbCR7ZtuZso1nhAXhWT8u", + "function": { + "name": "get_weather", + "arguments": "{\"city\": \"Paris\"}" + } + } + ], + "reasoning_details": [ + { + "type": "reasoning.text", + "text": "The user is asking me to:\n1. Think about which city is the capital of France\n2. Call get_weather for that city\n\nThe capital of France is Paris. So I should call get_weather with \"Paris\" as the city parameter.", + "format": "anthropic-claude-v1", + "index": 0, + "signature": "sig-fixture-1" + } + ] + } + } + ], + "usage": { + "prompt_tokens": 609, + "completion_tokens": 114, + "total_tokens": 723, + "prompt_tokens_details": { + "cached_tokens": 0, + "cache_write_tokens": 0, + "audio_tokens": 0, + "video_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 58, + "image_tokens": 0, + "audio_tokens": 0 + } + } +} diff --git a/src/lib/ai/__tests__/local-reasoning.test.ts b/src/lib/ai/__tests__/local-reasoning.test.ts new file mode 100644 index 000000000..5c0a27c65 --- /dev/null +++ b/src/lib/ai/__tests__/local-reasoning.test.ts @@ -0,0 +1,278 @@ +import { readFileSync } from "node:fs"; +import { join } from "node:path"; +import { beforeEach, describe, expect, it, vi } from "vitest"; + +// Destination policy has its own suites; stub it so the wire can be asserted +// against the global fetch stub. +vi.mock("../local-host-allowlist", () => ({ aiEgressPolicyFor: () => ({}) })); +import { + LocalOpenAICompatibleClient, + resetLocalJsonDialectCache, +} from "../local-client"; +import { resetReasoningDialectCache } from "../reasoning/dialect-cache"; +import type { AiReasoningEvent, CompletionParams } from "../types"; + +/** + * v1.41 — reasoning on local OpenAI-compatible servers. The reply shapes are + * the documented ones: `reasoning_content` (llama.cpp, LiteLLM), `reasoning` + * (vLLM, Ollama, LM Studio), on the message and on each streamed delta, and + * an inline leading `` block. + */ +const FIXTURES = join(__dirname, "fixtures/reasoning"); + +const BASE = "http://localhost:11434/v1"; + +function buffered(message: Record) { + return async () => + new Response( + JSON.stringify({ + choices: [{ message, finish_reason: "stop" }], + usage: { + total_tokens: 70, + completion_tokens_details: { reasoning_tokens: 30 }, + }, + }), + { status: 200, headers: { "content-type": "application/json" } }, + ); +} + +function stream(deltas: Array>) { + return async () => { + const encoder = new TextEncoder(); + const frames = [ + ...deltas.map( + (delta) => + `data: ${JSON.stringify({ choices: [{ delta, finish_reason: null }] })}\n\n`, + ), + `data: ${JSON.stringify({ choices: [{ delta: {}, finish_reason: "stop" }], usage: { total_tokens: 55 } })}\n\n`, + "data: [DONE]\n\n", + ]; + return new Response( + new ReadableStream({ + start(c) { + for (const f of frames) c.enqueue(encoder.encode(f)); + c.close(); + }, + }), + { status: 200, headers: { "content-type": "text/event-stream" } }, + ); + }; +} + +function params(extra: Partial = {}): CompletionParams { + return { + system: "s", + messages: [{ role: "user", content: "u" }], + maxTokens: 600, + ...extra, + }; +} + +const body = (mock: ReturnType, call = 0) => + JSON.parse(mock.mock.calls[call][1].body); + +const client = () => + new LocalOpenAICompatibleClient({ model: "qwen3:8b", baseUrl: BASE }); + +beforeEach(() => { + vi.restoreAllMocks(); + resetLocalJsonDialectCache(); + resetReasoningDialectCache(); +}); + +describe("Local reasoning request", () => { + it("sends the call's level, off as none, and gives it room to think", async () => { + const mock = vi.fn().mockImplementation(buffered({ content: "ok" })); + vi.stubGlobal("fetch", mock); + const c = client(); + c.reasoningEffort = "high"; + await c.generateCompletion(params()); + await c.generateCompletion( + params({ reasoning: { effort: "medium", summaries: true } }), + ); + await c.generateCompletion( + params({ reasoning: { effort: "off", summaries: true } }), + ); + // No level on the call: the entry's own setting, as before. + expect(body(mock, 0).reasoning_effort).toBe("high"); + expect(body(mock, 0).max_tokens).toBe(600); + expect(body(mock, 1).reasoning_effort).toBe("medium"); + expect(body(mock, 1).max_tokens).toBe(600 + 4096); + expect(body(mock, 2).reasoning_effort).toBe("none"); + expect(body(mock, 2).max_tokens).toBe(600); + }); + + it("retries once without the field when the server refuses it, and remembers", async () => { + const refusal = readFileSync( + join(FIXTURES, "gateway-reasoning-rejected.json"), + "utf8", + ); + const mock = vi + .fn() + .mockImplementationOnce( + async () => new Response(refusal, { status: 400 }), + ) + .mockImplementation(buffered({ content: "ok" })); + vi.stubGlobal("fetch", mock); + const c = client(); + const result = await c.generateCompletion( + params({ reasoning: { effort: "low", summaries: true } }), + ); + expect(result.content).toBe("ok"); + expect(body(mock, 1)).not.toHaveProperty("reasoning_effort"); + await c.generateCompletion( + params({ reasoning: { effort: "low", summaries: true } }), + ); + expect(mock).toHaveBeenCalledTimes(3); + expect(body(mock, 2)).not.toHaveProperty("reasoning_effort"); + }); +}); + +describe("Local reasoning replies", () => { + it.each([["reasoning_content"], ["reasoning"]])( + "reads %s off the message", + async (field) => { + vi.stubGlobal( + "fetch", + vi + .fn() + .mockImplementation( + buffered({ content: "Answer.", [field]: "**Plan**\nThink." }), + ), + ); + const events: AiReasoningEvent[] = []; + const result = await client().generateCompletion( + params({ + reasoning: { effort: "low", summaries: true }, + onReasoning: (e) => events.push(e), + }), + ); + expect(result.content).toBe("Answer."); + expect(result.reasoning).toEqual({ + summary: ["**Plan**\nThink."], + tokens: 30, + }); + expect(events).toEqual([ + { kind: "title", text: "Plan" }, + { kind: "text", text: "**Plan**\nThink." }, + { kind: "done", text: "" }, + ]); + }, + ); + + it("splits an inline think block only for a caller that asked for reasoning", async () => { + const reply = "check the window\n\nThe answer."; + vi.stubGlobal( + "fetch", + vi.fn().mockImplementation(buffered({ content: reply })), + ); + const asked = await client().generateCompletion( + params({ reasoning: { effort: "low", summaries: true } }), + ); + expect(asked.content).toBe("The answer."); + expect(asked.reasoning?.summary).toEqual(["check the window"]); + const unasked = await client().generateCompletion(params()); + expect(unasked.content).toBe(reply); + expect(unasked.reasoning).toBeUndefined(); + }); +}); + +describe("Local reasoning stream", () => { + it("reports streamed reasoning live and keeps it out of the answer", async () => { + vi.stubGlobal( + "fetch", + vi + .fn() + .mockImplementation( + stream([ + { reasoning_content: "**Choosing" }, + { reasoning_content: " the window**\n" }, + { reasoning_content: "Ninety days." }, + { content: "Your " }, + { content: "answer." }, + ]), + ), + ); + const events: AiReasoningEvent[] = []; + const deltas: string[] = []; + const result = await client().generateCompletionStream( + params({ + reasoning: { effort: "medium", summaries: true }, + onReasoning: (e) => events.push(e), + }), + (d) => deltas.push(d), + ); + expect(deltas).toEqual(["Your ", "answer."]); + expect(result.content).toBe("Your answer."); + expect(events).toEqual([ + { kind: "title", text: "Choosing the window" }, + { kind: "text", text: "**Choosing the window**\nNinety days." }, + { kind: "done", text: "" }, + ]); + expect(result.reasoning?.summary).toEqual([ + "**Choosing the window**\nNinety days.", + ]); + }); + + it("routes an inline think block split across chunks to the reasoning", async () => { + vi.stubGlobal( + "fetch", + vi + .fn() + .mockImplementation( + stream([ + { content: " weigh the" }, + { content: " two readings\n\nThe " }, + { content: "answer." }, + ]), + ), + ); + const deltas: string[] = []; + const result = await client().generateCompletionStream( + params({ reasoning: { effort: "low", summaries: true } }), + (d) => deltas.push(d), + ); + expect(deltas.join("")).toBe("The answer."); + expect(result.content).toBe("The answer."); + expect(result.reasoning?.summary).toEqual(["weigh the two readings"]); + }); + + it("streams untouched for a caller that asked for no reasoning", async () => { + vi.stubGlobal( + "fetch", + vi + .fn() + .mockImplementation( + stream([ + { reasoning: "ignored" }, + { content: "x" }, + { content: "y" }, + ]), + ), + ); + const deltas: string[] = []; + const onReasoning = vi.fn(); + const result = await client().generateCompletionStream( + params({ onReasoning }), + (d) => deltas.push(d), + ); + expect(deltas).toEqual(["x", "y"]); + expect(result.content).toBe("xy"); + expect(result.reasoning).toBeUndefined(); + expect(onReasoning).not.toHaveBeenCalled(); + }); + + it("passes a short reply that only looked like it might open a think block", async () => { + vi.stubGlobal( + "fetch", + vi.fn().mockImplementation(stream([{ content: " {}, + ); + expect(result.content).toBe(" { + const actual = await importOriginal(); + return { + ...actual, + fetch: (input: unknown, init?: unknown) => + (globalThis.fetch as unknown as (i: unknown, n?: unknown) => unknown)( + input, + init, + ), + }; +}); + +/** + * v1.41 — reasoning on the Chat Completions wire. The OpenRouter fixtures + * were recorded on 2026-10-06 with a throwaway prompt (no health data); + * signatures and encrypted payloads are placeholders. The gateway refusal is + * the standard OpenAI error envelope. + */ +const FIXTURES = join(__dirname, "fixtures/reasoning"); +const fixture = (name: string) => readFileSync(join(FIXTURES, name), "utf8"); +const ok = (name: string) => async () => + new Response(fixture(name), { + status: 200, + headers: { "content-type": "application/json" }, + }); +const plain = + (content: string, extra: Record = {}) => + async () => + new Response( + JSON.stringify({ + choices: [ + { + message: { role: "assistant", content, ...extra }, + finish_reason: "stop", + }, + ], + usage: { + total_tokens: 90, + completion_tokens_details: { reasoning_tokens: 40 }, + }, + }), + { status: 200 }, + ); + +const TOOLS: AiToolDef[] = [ + { + name: "get_weather", + description: "Weather for a city", + parameters: { type: "object", properties: { city: { type: "string" } } }, + }, +]; + +function params(extra: Partial = {}): CompletionParams { + return { + system: "You are terse.", + messages: [{ role: "user", content: "Weather in the capital of France?" }], + maxTokens: 600, + ...extra, + }; +} + +const body = (mock: ReturnType, call = 0) => + JSON.parse(mock.mock.calls[call][1].body); + +const openai = (model: string) => + new OpenAIClient({ + apiKey: "sk-test", + model, + baseUrl: "https://api.openai.com/v1", + }); +const gateway = (baseUrl: string, model: string) => + new OpenAIClient({ + apiKey: "sk-test", + model, + baseUrl, + providerType: "openai-compatible", + }); + +beforeEach(() => { + vi.restoreAllMocks(); + resetReasoningDialectCache(); + resetJsonModeDialectCache(); +}); + +describe("OpenAI reasoning", () => { + it("sends nothing new when the call asks for no reasoning", async () => { + const mock = vi.fn().mockImplementation(plain("ok")); + vi.stubGlobal("fetch", mock); + const result = await openai("gpt-5.5").generateCompletion(params()); + expect(body(mock)).not.toHaveProperty("reasoning_effort"); + expect(body(mock).max_completion_tokens).toBe(600); + expect(result.reasoning).toBeUndefined(); + }); + + it("sends the effort on a round without tools, with room to think", async () => { + const mock = vi.fn().mockImplementation(plain("ok")); + vi.stubGlobal("fetch", mock); + const result = await openai("gpt-5.5").generateCompletion( + params({ reasoning: { effort: "medium", summaries: true } }), + ); + expect(body(mock).reasoning_effort).toBe("medium"); + expect(body(mock).max_completion_tokens).toBe(600 + 4096); + expect(body(mock)).not.toHaveProperty("temperature"); + expect(result.reasoning).toEqual({ summary: [], tokens: 40 }); + }); + + it("sends no effort on a tool round", async () => { + const mock = vi.fn().mockImplementation(plain("ok")); + vi.stubGlobal("fetch", mock); + await openai("gpt-5.5").generateCompletion( + params({ tools: TOOLS, reasoning: { effort: "high", summaries: true } }), + ); + expect(body(mock)).not.toHaveProperty("reasoning_effort"); + expect(body(mock).max_completion_tokens).toBe(600); + }); + + it("maps off onto each family's lowest effort, and skips non-reasoning models", async () => { + const mock = vi.fn().mockImplementation(plain("ok")); + vi.stubGlobal("fetch", mock); + const off = { reasoning: { effort: "off" as const, summaries: false } }; + await openai("gpt-5.5").generateCompletion(params(off)); + await openai("gpt-5-mini").generateCompletion(params(off)); + await openai("o3").generateCompletion(params(off)); + await openai("gpt-4o").generateCompletion(params(off)); + expect(body(mock, 0).reasoning_effort).toBe("none"); + expect(body(mock, 1).reasoning_effort).toBe("minimal"); + expect(body(mock, 2).reasoning_effort).toBe("low"); + expect(body(mock, 3)).not.toHaveProperty("reasoning_effort"); + }); +}); + +describe("Gateway reasoning", () => { + it("lets the call's level win over the entry's stored setting", async () => { + const mock = vi.fn().mockImplementation(plain("ok")); + vi.stubGlobal("fetch", mock); + const c = gateway("https://litellm.example.com/v1", "qwen3"); + c.reasoningEffort = "high"; + await c.generateCompletion(params()); + await c.generateCompletion( + params({ reasoning: { effort: "low", summaries: true } }), + ); + expect(body(mock, 0).reasoning_effort).toBe("high"); + expect(body(mock, 1).reasoning_effort).toBe("low"); + }); + + it("reads reasoning_content as the summary", async () => { + vi.stubGlobal( + "fetch", + vi + .fn() + .mockImplementation( + plain("Paris.", { reasoning_content: "**Capital**\nIt is Paris." }), + ), + ); + const events: AiReasoningEvent[] = []; + const result = await gateway( + "https://litellm.example.com/v1", + "qwen3", + ).generateCompletion( + params({ + reasoning: { effort: "medium", summaries: true }, + onReasoning: (e) => events.push(e), + }), + ); + expect(result.content).toBe("Paris."); + expect(result.reasoning?.summary).toEqual(["**Capital**\nIt is Paris."]); + expect(events.map((e) => e.kind)).toEqual(["title", "text", "done"]); + }); + + it("speaks OpenRouter's dialect and carries reasoning_details to the next round", async () => { + const mock = vi + .fn() + .mockImplementationOnce(ok("openrouter-claude-tool-round.json")) + .mockImplementation(ok("openrouter-claude-after-tool.json")); + vi.stubGlobal("fetch", mock); + const c = gateway( + "https://openrouter.ai/api/v1", + "anthropic/claude-haiku-4.5", + ); + const first = await c.generateCompletion( + params({ tools: TOOLS, reasoning: { effort: "low", summaries: true } }), + ); + expect(body(mock, 0).reasoning).toEqual({ effort: "low" }); + expect(body(mock, 0)).not.toHaveProperty("reasoning_effort"); + expect(first.finishReason).toBe("tool_calls"); + expect(first.reasoning?.tokens).toBe(58); + expect(first.reasoning?.summary[0]).toMatch(/capital of France is Paris/); + const details = JSON.parse(fixture("openrouter-claude-tool-round.json")) + .choices[0].message.reasoning_details; + expect(first.providerState).toEqual({ + providerType: "openai-compatible", + model: "anthropic/claude-haiku-4.5", + items: [{ reasoning_details: details }], + }); + + await c.generateCompletion( + params({ + tools: TOOLS, + reasoning: { effort: "low", summaries: true }, + messages: [ + { role: "user", content: "Weather in the capital of France?" }, + { + role: "assistant", + content: first.content, + toolCalls: first.toolCalls, + providerState: first.providerState, + }, + { role: "tool", toolCallId: first.toolCalls![0].id, content: "{}" }, + ], + }), + ); + const assistant = body(mock, 1).messages[2]; + expect(assistant.role).toBe("assistant"); + expect(assistant.reasoning_details).toEqual(details); + }); + + it("does not hand state to a different model", async () => { + const mock = vi.fn().mockImplementation(plain("ok")); + vi.stubGlobal("fetch", mock); + await gateway( + "https://openrouter.ai/api/v1", + "openai/gpt-5-mini", + ).generateCompletion( + params({ + messages: [ + { role: "user", content: "q" }, + { + role: "assistant", + content: "", + toolCalls: [{ id: "c1", name: "get_weather", arguments: "{}" }], + providerState: { + providerType: "openai-compatible", + model: "anthropic/claude-haiku-4.5", + items: [{ reasoning_details: [{ type: "reasoning.text" }] }], + }, + }, + { role: "tool", toolCallId: "c1", content: "{}" }, + ], + }), + ); + expect(body(mock).messages[2]).not.toHaveProperty("reasoning_details"); + }); + + it("retries once without reasoning when the gateway refuses it, and remembers", async () => { + const mock = vi + .fn() + .mockImplementationOnce( + async () => + new Response(fixture("gateway-reasoning-rejected.json"), { + status: 400, + }), + ) + .mockImplementation(plain("ok")); + vi.stubGlobal("fetch", mock); + const c = gateway("https://litellm.example.com/v1", "llama3"); + const result = await c.generateCompletion( + params({ + responseFormat: "json", + reasoning: { effort: "medium", summaries: true }, + }), + ); + expect(body(mock, 0).reasoning_effort).toBe("medium"); + expect(body(mock, 1)).not.toHaveProperty("reasoning_effort"); + // The JSON flag survives: the refusal named the reasoning field. + expect(body(mock, 1).response_format).toEqual({ type: "json_object" }); + expect(jsonModeDialectFor("https://litellm.example.com/v1")).toBe( + "response_format", + ); + expect(result.reasoning?.downgradedTo).toBe("off"); + + await c.generateCompletion( + params({ reasoning: { effort: "medium", summaries: true } }), + ); + expect(mock).toHaveBeenCalledTimes(3); + expect(body(mock, 2)).not.toHaveProperty("reasoning_effort"); + }); +}); diff --git a/src/lib/ai/local-client.ts b/src/lib/ai/local-client.ts index e753050ef..2f6ef99e6 100644 --- a/src/lib/ai/local-client.ts +++ b/src/lib/ai/local-client.ts @@ -24,6 +24,13 @@ import { type JsonModeDialect, } from "./json-dialect"; import type { ReasoningEffort } from "./reasoning-effort"; +import { + annotateReasoningDowngrade, + learnedReasoningDialect, + rememberReasoningDialect, +} from "./reasoning/dialect-cache"; +import { REASONING_THINKING_BUDGET } from "./reasoning/levels"; +import { reasoningTitleOf, splitThinkTags } from "./reasoning/support"; interface LocalClientConfig { apiKey?: string | null; @@ -88,6 +95,68 @@ function withStrictJsonPrefix(messages: AiMessage[]): AiMessage[] { return out; } +/** + * v1.41 — collects what a call reasoned and reports it to `onReasoning`. + * Local servers return the reasoning as `reasoning_content` (llama.cpp, + * LiteLLM) or `reasoning` (vLLM, Ollama, LM Studio), on the message and on + * each streamed delta, or inline as a leading `…` block. + * https://github.com/ggml-org/llama.cpp/blob/master/tools/server/README.md, + * https://docs.vllm.ai/en/latest/features/reasoning_outputs.html, + * https://docs.ollama.com/api/openai-compatibility + */ +class ReasoningCollector { + private text = ""; + private titled = false; + private finished = false; + constructor(private readonly params: CompletionParams) {} + + private get emit() { + return this.params.reasoning ? this.params.onReasoning : undefined; + } + + /** A streamed piece of reasoning; the title goes out once it has closed. */ + add(delta: string): void { + this.text += delta; + if (!this.titled && this.emit) { + const title = reasoningTitleOf(this.text); + if (title) { + this.titled = true; + this.emit({ kind: "title", text: title }); + } + } + } + + /** The reasoning is complete (the answer started, or the call ended). */ + finish(): void { + if (this.finished) return; + this.finished = true; + const text = this.text.trim(); + if (!text || !this.emit) return; + if (!this.titled) { + const title = reasoningTitleOf(text); + if (title) this.emit({ kind: "title", text: title }); + } + this.emit({ kind: "text", text }); + } + + result(tokens: number | null): CompletionResult["reasoning"] { + this.finish(); + if (!this.params.reasoning) return undefined; + this.emit?.({ kind: "done", text: "" }); + const text = this.text.trim(); + return { summary: text ? [text] : [], tokens }; + } +} + +/** The reasoning text a non-streamed message carries, if any. */ +function messageReasoning(message: unknown): string | null { + const m = (message ?? {}) as Record; + const text = [m.reasoning_content, m.reasoning].find( + (v): v is string => typeof v === "string" && v.trim().length > 0, + ); + return text ?? null; +} + /** * Talks to OpenAI-compatible servers: local runtimes (Ollama, LocalAI, * LM Studio, vLLM, …) AND hosted gateways (LiteLLM, OpenRouter, any @@ -158,11 +227,18 @@ export class LocalOpenAICompatibleClient implements AIProvider { withStrictJsonPrefix(params.messages), ); + const callEffort = this.callReasoningEffort(params); + // Reasoning tokens count against `max_tokens` on these servers; the + // thinking budget rides on top of the answer budget (a ceiling only). + const thinkingTokens = + params.reasoning && callEffort && params.reasoning.effort !== "off" + ? REASONING_THINKING_BUDGET[params.reasoning.effort] + : 0; const body = JSON.stringify({ model: this.config.model, messages, temperature: params.temperature ?? 0.3, - max_tokens: params.maxTokens ?? 1000, + max_tokens: (params.maxTokens ?? 1000) + thinkingTokens, // Deterministic seed for reproducible reference output. Ollama + // most OpenAI-compatible local servers honour it; servers that // ignore it simply disregard the field. Omitted when unset. @@ -170,9 +246,11 @@ export class LocalOpenAICompatibleClient implements AIProvider { // #1126 — the entry's reasoning setting. Default sends nothing, so the // body of a setup that never touched it stays byte-identical; "Off" is // `none`, which stops a thinking model spending the answer budget. - ...(this.reasoningEffort + // v1.41 — a call that carries its own resolved level wins over it. + ...(!params.reasoning && this.reasoningEffort ? { reasoning_effort: this.reasoningEffort } : {}), + ...(callEffort ? { reasoning_effort: callEffort } : {}), // v1.28.28 (#470) — the JSON surfaces opt in via `responseFormat: // "json"`. Default dialect sends the STANDARD OpenAI field (the // top-level Ollama-native `format: "json"` this client used to send @@ -194,6 +272,27 @@ export class LocalOpenAICompatibleClient implements AIProvider { return { url, headers, body, jsonDialect }; } + private reasoningDialectKey() { + return { + provider: "local", + endpoint: this.config.baseUrl, + model: this.config.model, + }; + } + + /** + * The `reasoning_effort` a call with its own resolved level sends (`off` + * is `none`), unless this endpoint refused the field before. `null` for a + * call without one: the entry's #1126 setting applies, exactly as before. + */ + private callReasoningEffort(params: CompletionParams): string | null { + if (!params.reasoning) return null; + if (learnedReasoningDialect(this.reasoningDialectKey()) === "none") { + return null; + } + return params.reasoning.effort === "off" ? "none" : params.reasoning.effort; + } + async generateCompletion( params: CompletionParams, ): Promise { @@ -243,6 +342,20 @@ export class LocalOpenAICompatibleClient implements AIProvider { // strip anything that looks like a key before logging. const rawBody = await res.text().catch(() => ""); const bodyExcerpt = sanitiseBodyExcerpt(rawBody); + // v1.41 — a server that refuses the reasoning field is asked once more + // without it and remembers. Before the JSON dialect, so a complaint + // about reasoning never switches JSON mode off. + const sentEffort = this.callReasoningEffort(params); + if ( + sentEffort && + res.status >= 400 && + res.status < 500 && + /reasoning|think/i.test(bodyExcerpt) + ) { + rememberReasoningDialect(this.reasoningDialectKey(), "none"); + annotateReasoningDowngrade("local", sentEffort, "none"); + return this.generateCompletion(params); + } // v1.28.28 (#470) — dialect self-heal. A 4xx whose body names // `response_format` (or an unknown parameter) means this endpoint // rejects the standard JSON flag: learn the no-flag dialect for the @@ -292,7 +405,10 @@ export class LocalOpenAICompatibleClient implements AIProvider { message?: { content?: string }; finish_reason?: string; }>; - usage?: { total_tokens?: number }; + usage?: { + total_tokens?: number; + completion_tokens_details?: { reasoning_tokens?: number }; + }; }; // Same embedded-failure shape the OpenAI-compatible gateway tag handles: @@ -307,7 +423,22 @@ export class LocalOpenAICompatibleClient implements AIProvider { model: this.config.model, }); - const content = json.choices?.[0]?.message?.content; + const collector = new ReasoningCollector(params); + let content = json.choices?.[0]?.message?.content; + if (params.reasoning) { + const fielded = messageReasoning(json.choices?.[0]?.message); + if (fielded) collector.add(fielded); + // Only a caller that asked for reasoning gets the inline block split + // off; every other caller keeps the text exactly as it came. + if (content) { + const split = splitThinkTags(content); + if (split.reasoning) collector.add(split.reasoning); + content = split.content; + } + } + const reasoning = collector.result( + json.usage?.completion_tokens_details?.reasoning_tokens ?? null, + ); if (!content) { // v1.20.1 — sentinel httpStatus + kind so the chain classifier can tell // an empty 200-OK reply apart from a transport failure. Cascade unchanged @@ -329,6 +460,7 @@ export class LocalOpenAICompatibleClient implements AIProvider { // v1.28.28 (#470) — surface why the model stopped so the JSON callers // can tell a token-ceiling truncation ("length") from bad output. finishReason: mapFinishReason(json.choices?.[0]?.finish_reason), + ...(reasoning ? { reasoning } : {}), }; } @@ -416,7 +548,18 @@ export class LocalOpenAICompatibleClient implements AIProvider { label: "local AI endpoint", model: this.config.model, }); - const buffered = json?.choices?.[0]?.message?.content; + const collector = new ReasoningCollector(params); + let buffered = json?.choices?.[0]?.message?.content; + if (params.reasoning) { + const fielded = messageReasoning(json?.choices?.[0]?.message); + if (fielded) collector.add(fielded); + if (buffered) { + const split = splitThinkTags(buffered); + if (split.reasoning) collector.add(split.reasoning); + buffered = split.content; + } + } + const reasoning = collector.result(null); if (!buffered) { const err = new Error("Local AI returned empty content"); Object.assign(err, { @@ -433,6 +576,7 @@ export class LocalOpenAICompatibleClient implements AIProvider { model: this.config.model, providerType: "local", finishReason: mapFinishReason(json?.choices?.[0]?.finish_reason), + ...(reasoning ? { reasoning } : {}), }; } @@ -441,8 +585,55 @@ export class LocalOpenAICompatibleClient implements AIProvider { let sseBuffer = ""; let content = ""; let tokensUsed: number | null = null; + let reasoningTokens: number | null = null; let finishReason: string | undefined; + // v1.41 — reasoning on the stream: the `reasoning_content` / `reasoning` + // delta fields, or a leading `` block inside `content`. Inline + // reasoning never reaches `onDelta`; it would otherwise show up as the + // answer. Only for a caller that asked for reasoning. + const collector = new ReasoningCollector(params); + let thinkState: "detect" | "inside" | "passthrough" = params.reasoning + ? "detect" + : "passthrough"; + let pending = ""; + const passContent = (text: string) => { + if (!text) return; + collector.finish(); + content += text; + onDelta(text); + }; + const onContent = (delta: string) => { + if (thinkState === "passthrough") { + passContent(delta); + return; + } + pending += delta; + if (thinkState === "detect") { + const lead = pending.replace(/^\s+/, ""); + if (lead.length < "".length && "".startsWith(lead)) { + return; + } + if (!lead.startsWith("")) { + thinkState = "passthrough"; + const out = pending; + pending = ""; + passContent(out); + return; + } + thinkState = "inside"; + pending = lead.slice("".length); + } + const close = pending.indexOf(""); + if (close === -1) return; + collector.add(pending.slice(0, close)); + collector.finish(); + const rest = pending.slice(close + "".length).replace(/^\s+/, ""); + pending = ""; + thinkState = "passthrough"; + passContent(rest); + }; + // An SSE frame can carry the same embedded failure the buffered body // carries. Remember the first one; it only matters if the stream ends // without a single token, in which case it is the honest reason. @@ -474,11 +665,18 @@ export class LocalOpenAICompatibleClient implements AIProvider { let chunk: { error?: unknown; choices?: Array<{ - delta?: { content?: string }; + delta?: { + content?: string; + reasoning_content?: string; + reasoning?: string; + }; finish_reason?: string | null; error?: unknown; }>; - usage?: { total_tokens?: number }; + usage?: { + total_tokens?: number; + completion_tokens_details?: { reasoning_tokens?: number }; + }; }; try { chunk = JSON.parse(payload); @@ -489,14 +687,19 @@ export class LocalOpenAICompatibleClient implements AIProvider { if (streamedError === null && readEmbeddedError(chunk) !== null) { streamedError = chunk; } - const delta = chunk.choices?.[0]?.delta?.content; - if (delta) { - content += delta; - onDelta(delta); + const choiceDelta = chunk.choices?.[0]?.delta; + const reasoningDelta = + choiceDelta?.reasoning_content ?? choiceDelta?.reasoning; + if (params.reasoning && typeof reasoningDelta === "string") { + collector.add(reasoningDelta); } + const delta = choiceDelta?.content; + if (delta) onContent(delta); if (typeof chunk.usage?.total_tokens === "number") { tokensUsed = chunk.usage.total_tokens; } + const rt = chunk.usage?.completion_tokens_details?.reasoning_tokens; + if (typeof rt === "number") reasoningTokens = rt; const chunkFinish = chunk.choices?.[0]?.finish_reason; if (typeof chunkFinish === "string") finishReason = chunkFinish; } @@ -521,6 +724,19 @@ export class LocalOpenAICompatibleClient implements AIProvider { reader.cancel().catch(() => {}); } + // A reply short enough to still sit in the `` detector is prose; + // an unclosed block is reasoning that ran out of budget. + // (The closures above move `thinkState`; read it afresh.) + const endState = thinkState as "detect" | "inside" | "passthrough"; + if (endState === "detect") { + thinkState = "passthrough"; + passContent(pending); + } else if (endState === "inside") { + collector.add(pending); + } + pending = ""; + const reasoning = collector.result(reasoningTokens); + if (!content) { // A stream that ended with a failure frame and no token reports the // failure, not "empty content". @@ -544,6 +760,7 @@ export class LocalOpenAICompatibleClient implements AIProvider { model: this.config.model, providerType: "local", finishReason: mapFinishReason(finishReason), + ...(reasoning ? { reasoning } : {}), }; } } diff --git a/src/lib/ai/openai-client.ts b/src/lib/ai/openai-client.ts index f54ea1bf1..e8acd00bd 100644 --- a/src/lib/ai/openai-client.ts +++ b/src/lib/ai/openai-client.ts @@ -2,7 +2,23 @@ import { safeFetch } from "@/lib/safe-fetch"; import { callTimeoutMs } from "./effective-timeout"; import { annotate } from "@/lib/logging/context"; import type { AIProvider, CompletionParams, CompletionResult } from "./types"; -import { selectOpenAIChatCompletionsCapabilities } from "./openai-capabilities"; +import { + selectOpenAIChatCompletionsCapabilities, + type OpenAIChatCompletionsCapabilities, +} from "./openai-capabilities"; +import { + annotateReasoningDowngrade, + learnedReasoningDialect, + rememberReasoningDialect, +} from "./reasoning/dialect-cache"; +import { REASONING_THINKING_BUDGET } from "./reasoning/levels"; +import { + isOpenRouterEndpoint, + levelOfWireEffort, + openAIOffEffort, + openAIReasoningFamily, + reasoningTitleOf, +} from "./reasoning/support"; import { hasLearnedJsonModeDialect, isResponseFormatRejection, @@ -59,6 +75,26 @@ interface OpenAIClientConfig { type OpenAIProviderType = "admin-key" | "codex" | "openai-compatible"; +/** + * v1.41 — the reasoning one request sends. + * - `reasoning_effort`: the Chat Completions field (OpenAI, LiteLLM, vLLM, + * most gateways). https://platform.openai.com/docs/api-reference/chat/create + * - `openrouter`: OpenRouter's `reasoning: { effort }` object, which also + * returns `reasoning_details` to hand back on tool rounds. + * https://openrouter.ai/docs/use-cases/reasoning-tokens + */ +type OpenAIReasoningWire = + | { kind: "none" } + | { kind: "reasoning_effort"; effort: string } + | { kind: "openrouter"; effort: string }; + +/** + * Assistant-message fields a gateway returns and needs back unmodified on the + * next tool round: OpenRouter's `reasoning_details` and the `thinking_blocks` + * LiteLLM carries for Claude upstreams (https://docs.litellm.ai/docs/reasoning_content). + */ +const ROUND_TRIP_FIELDS = ["reasoning_details", "thinking_blocks"] as const; + export class OpenAIClient implements AIProvider { readonly type: OpenAIProviderType; /** See `AIProvider.responseTimeoutSeconds`; stamped by the resolver. */ @@ -94,7 +130,10 @@ export class OpenAIClient implements AIProvider { // conversation turns mapped 1:1. Vision parts (Lab-OCR) become the // multimodal `image_url` content array `gpt-4o`-class models accept. The // image is framed as untrusted DATA by the system prompt. - const messages = buildOpenAIMessages(params.system, params.messages); + const messages = this.withRoundTripState( + buildOpenAIMessages(params.system, params.messages), + params, + ); // v1.20.0 — tool plumbing. The defs map onto the OpenAI `function` tool // wire; F1 supplies real defs and consumes the parsed `toolCalls`. No F4 @@ -122,7 +161,19 @@ export class OpenAIClient implements AIProvider { this.config.baseUrl, this.config.model, ); - const tokenBudget = params.maxTokens ?? 1000; + const reasoningWire = this.planReasoning(params, hasTools, capabilities); + // Reasoning tokens count against the completion budget on every one of + // these wires, so the thinking budget rides on top of the answer budget + // (a ceiling, not a charge). Without it a 600-token Coach round spends + // its whole allowance thinking and returns no answer. + const wireLevel = + reasoningWire.kind === "none" || reasoningWire.effort === "none" + ? null + : levelOfWireEffort(reasoningWire.effort); + const thinkingTokens = wireLevel + ? REASONING_THINKING_BUDGET[wireLevel === "off" ? "low" : wireLevel] + : 0; + const tokenBudget = (params.maxTokens ?? 1000) + thinkingTokens; const capabilityParams = capabilities.supportsSamplingControls ? { [capabilities.tokenBudgetField]: tokenBudget, @@ -157,10 +208,16 @@ export class OpenAIClient implements AIProvider { ...(tools ? { tools } : {}), ...(params.toolChoice ? { tool_choice: params.toolChoice } : {}), // #1126 — the gateway entry's reasoning setting; Default sends - // nothing. `api.openai.com` and Codex never get it. - ...(this.isGateway && this.reasoningEffort + // nothing. `api.openai.com` and Codex never get it. v1.41 — a call + // that carries its own resolved level (`params.reasoning`) wins. + ...(!params.reasoning && this.isGateway && this.reasoningEffort ? { reasoning_effort: this.reasoningEffort } : {}), + ...(reasoningWire.kind === "reasoning_effort" + ? { reasoning_effort: reasoningWire.effort } + : reasoningWire.kind === "openrouter" + ? { reasoning: { effort: reasoningWire.effort } } + : {}), }), }, // 60 s ceiling so a tar-pit upstream cannot pin a worker @@ -205,6 +262,20 @@ export class OpenAIClient implements AIProvider { // that looks like an API key from the excerpt before logging. const rawBody = await res.text().catch(() => ""); const bodyExcerpt = sanitiseBodyExcerpt(rawBody); + // v1.41 — an endpoint or model that refuses the reasoning field is + // asked once more without it and remembers, so the answer never fails + // on a reasoning parameter. Checked before the JSON dialect: a body + // naming the reasoning field must not switch JSON mode off. + if ( + reasoningWire.kind !== "none" && + res.status >= 400 && + res.status < 500 && + /reasoning|thinking/i.test(bodyExcerpt) + ) { + rememberReasoningDialect(this.reasoningDialectKey(), "none"); + annotateReasoningDowngrade(this.type, reasoningWire.effort, "none"); + return this.generateCompletion(params); + } // v1.33.1 (#470) — dialect self-heal, gateway only. A 4xx whose body // names `response_format` (or an unknown parameter) means this endpoint // rejects the standard JSON flag: learn the no-flag dialect for the base @@ -320,6 +391,31 @@ export class OpenAIClient implements AIProvider { throw err; } + // v1.41 — what the call reasoned. Gateways return the text as + // `reasoning` (OpenRouter, vLLM, Ollama) or `reasoning_content` + // (LiteLLM, llama.cpp); `api.openai.com` returns none on this wire. + const message = (choice?.message ?? {}) as Record; + const reasoningText = [message.reasoning, message.reasoning_content].find( + (v): v is string => typeof v === "string" && v.trim().length > 0, + ); + const summary = reasoningText ? [reasoningText.trim()] : []; + const emit = params.reasoning ? params.onReasoning : undefined; + if (emit) { + for (const text of summary) { + const title = reasoningTitleOf(text); + if (title) emit({ kind: "title", text: title }); + emit({ kind: "text", text }); + } + emit({ kind: "done", text: "" }); + } + const roundTrip = ROUND_TRIP_FIELDS.filter( + (field) => message[field] !== undefined && message[field] !== null, + ).map((field) => ({ [field]: message[field] })); + const usage = json.usage as + { completion_tokens_details?: { reasoning_tokens?: number } } | undefined; + const learnedNone = + learnedReasoningDialect(this.reasoningDialectKey()) === "none"; + return { content: content ?? "", tokensUsed: json.usage?.total_tokens ?? null, @@ -328,6 +424,116 @@ export class OpenAIClient implements AIProvider { providerType: this.type, ...(toolCalls ? { toolCalls } : {}), finishReason: mapFinishReason(choice?.finish_reason), + ...(params.reasoning + ? { + reasoning: { + summary, + // Billed as completion tokens, already inside `total_tokens`. + tokens: + usage?.completion_tokens_details?.reasoning_tokens ?? null, + ...(learnedNone && params.reasoning.effort !== "off" + ? { downgradedTo: "off" as const } + : {}), + }, + } + : {}), + ...(roundTrip.length > 0 + ? { + providerState: { + providerType: this.type, + model: this.config.model, + items: roundTrip, + }, + } + : {}), }; } + + // ── v1.41 reasoning ─────────────────────────────────────────────────────── + + private reasoningDialectKey() { + return { + provider: "openai", + endpoint: this.config.baseUrl, + model: this.config.model, + }; + } + + /** + * The reasoning this request sends. Absent `params.reasoning` → none here + * (the #1126 gateway setting is applied separately, unchanged). + * + * `api.openai.com` (operator key, personal key): `reasoning_effort` only for + * a reasoning model, and only on a round without tools. Chat Completions + * has no reasoning items to hand back between tool rounds, and newer + * models restrict reasoning alongside function calls; the Coach's final + * round and the background jobs carry no tools, which is where it counts. + * + * Gateway: the endpoint's own dialect, tools or not; its reasoning state + * travels in `providerState`, and a refusal is learned per endpoint. + */ + private planReasoning( + params: CompletionParams, + hasTools: boolean, + capabilities: OpenAIChatCompletionsCapabilities, + ): OpenAIReasoningWire { + const reasoning = params.reasoning; + if (!reasoning) return { kind: "none" }; + if (learnedReasoningDialect(this.reasoningDialectKey()) === "none") { + return { kind: "none" }; + } + const level = reasoning.effort; + if (this.isGateway) { + const effort = level === "off" ? "none" : level; + return isOpenRouterEndpoint(this.config.baseUrl) + ? { kind: "openrouter", effort } + : { kind: "reasoning_effort", effort }; + } + const family = openAIReasoningFamily(this.config.model); + if ( + !family || + hasTools || + capabilities.tokenBudgetField !== "max_completion_tokens" + ) { + return { kind: "none" }; + } + return { + kind: "reasoning_effort", + effort: level === "off" ? openAIOffEffort(family) : level, + }; + } + + /** + * Put the reasoning state of earlier tool rounds back on their assistant + * messages. `buildOpenAIMessages` maps turn `i` to wire message `i + 1` + * (the system turn leads). Only state this client made for this model is + * used; anything else is dropped. + */ + private withRoundTripState( + wire: T[], + params: CompletionParams, + ): T[] { + params.messages.forEach((m, i) => { + const state = m.providerState; + if ( + m.role !== "assistant" || + !state || + state.providerType !== this.type || + state.model !== this.config.model + ) { + return; + } + const target = wire[i + 1] as Record | undefined; + if (!target) return; + for (const item of state.items) { + if (!item || typeof item !== "object") continue; + for (const field of ROUND_TRIP_FIELDS) { + if (field in item) { + target[field] = (item as Record)[field]; + } + } + } + }); + return wire; + } } From edf6fffc32a18bfb8cdc185af4660f6bc672a568 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Tue, 6 Oct 2026 19:26:30 +0200 Subject: [PATCH 5/7] test: admit the per-call reasoning level in the reasoning_effort guard The key now appears twice in each client: the bound chain setting, sent only when the call carries no level, and the call's resolved level. The guard pins both, the condition in front of the bound value, and the absence of a literal. --- .../ai-reasoning-effort-guard.test.ts | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/src/__tests__/ai-reasoning-effort-guard.test.ts b/src/__tests__/ai-reasoning-effort-guard.test.ts index e555ab141..2c4fac64e 100644 --- a/src/__tests__/ai-reasoning-effort-guard.test.ts +++ b/src/__tests__/ai-reasoning-effort-guard.test.ts @@ -3,11 +3,15 @@ * the setting the resolver bound to the provider instance. * * Default must send nothing, so the key may not appear on any wire except as - * the bound value, and only the two clients whose endpoint a person chose may - * carry it. Pinned here: + * the bound value or the call's own resolved level, and only the two clients + * whose endpoint a person chose may carry it. Pinned here: * * 1. the key appears in exactly the Local client and the OpenAI client, - * once each, and only as `reasoning_effort: this.reasoningEffort`; + * twice each: once as `reasoning_effort: this.reasoningEffort`, sent + * only when the call carries no level of its own (`!params.reasoning`), + * and once as the call's level (v1.41 `CompletionParams.reasoning`, + * resolved upstream against the person's choice and the operator's + * cap), never as a literal; * 2. nothing but the binder assigns `.reasoningEffort`; * 3. every exported resolver in `provider.ts` calls `bindReasoningEffort`. * @@ -20,6 +24,7 @@ * the behaviour on the wire. * * Mutation check: hard-code `reasoning_effort: "none"` into either client, + * drop the `!params.reasoning` condition in front of the bound value, * set `client.reasoningEffort = ...` in a route, or drop `bindReasoningEffort` * from `resolveProviderForTest`: each one goes red here. */ @@ -60,13 +65,17 @@ describe("reasoning_effort on the wire", () => { }); it.each(["lib/ai/local-client.ts", "lib/ai/openai-client.ts"])( - "%s sends it once, as the bound value", + "%s sends the bound value only without a call level, and never a literal", (rel) => { const source = code(rel); - expect(source.match(new RegExp(KEY))?.length).toBe(1); + expect(source.match(new RegExp(KEY))?.length).toBe(2); expect(source).toMatch( /\breasoning_effort\s*:\s*this\s*\.\s*reasoningEffort\b/, ); + expect(source).toMatch( + /!\s*params\s*\.\s*reasoning\s*&&\s*(?:this\s*\.\s*isGateway\s*&&\s*)?this\s*\.\s*reasoningEffort\b/, + ); + expect(source).not.toMatch(/\breasoning_effort["']?\s*:\s*["'`]/); }, ); }); From 34afa3a4f0ce63e4be53a7ff9ef0ff81b6b1786a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Tue, 6 Oct 2026 19:26:30 +0200 Subject: [PATCH 6/7] feat(ai): reason deterministically in the mock and count reasoning in the chain runner The mock reports a fixed summary, title and provider state for calls that ask for reasoning. The runner passes reasoning and onReasoning to every candidate unchanged and adds the reasoning token and summary counts to the wide event, never the text or the state. --- .../__tests__/reasoning-passthrough.test.ts | 128 ++++++++++++++++++ src/lib/ai/mock-client.ts | 47 +++++++ src/lib/ai/provider-runner.ts | 14 ++ 3 files changed, 189 insertions(+) create mode 100644 src/lib/ai/__tests__/reasoning-passthrough.test.ts diff --git a/src/lib/ai/__tests__/reasoning-passthrough.test.ts b/src/lib/ai/__tests__/reasoning-passthrough.test.ts new file mode 100644 index 000000000..16886899f --- /dev/null +++ b/src/lib/ai/__tests__/reasoning-passthrough.test.ts @@ -0,0 +1,128 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; +import type { AiReasoningEvent, CompletionParams } from "../types"; + +/** + * v1.41 — the chain runner hands `reasoning` and `onReasoning` to every + * candidate unchanged, reports reasoning as counts only, and the mock client + * reasons deterministically for the loop's tests. + */ +const annotations = vi.hoisted(() => [] as Array>); +vi.mock("@/lib/logging/context", () => ({ + annotate: (fields: { meta?: Record }) => { + if (fields.meta) annotations.push(fields.meta); + }, +})); +vi.mock("../coach/budget", async () => { + const actual = + await vi.importActual("../coach/budget"); + return { + ...actual, + readDailySpend: vi.fn(async () => ({ total: 0, operator: 0 })), + }; +}); + +import { MockAIProvider } from "../mock-client"; +import { + clearLastWorkingProviderCache, + runRawCompletionWithFallback, +} from "../provider-runner"; +import { createInMemoryProviderHealthLedger } from "../provider-health-ledger"; + +function params(extra: Partial = {}): CompletionParams { + return { + system: "s", + messages: [{ role: "user", content: "u" }], + ...extra, + }; +} + +beforeEach(() => { + annotations.length = 0; + clearLastWorkingProviderCache(); +}); + +describe("MockAIProvider reasoning", () => { + it("reasons deterministically when asked, with state for the next round", async () => { + const mock = new MockAIProvider({ providerType: "codex", responses: "a" }); + const events: AiReasoningEvent[] = []; + const first = await mock.generateCompletion( + params({ + reasoning: { effort: "medium", summaries: true }, + onReasoning: (e) => events.push(e), + }), + ); + expect(events).toEqual([ + { kind: "title", text: "Reading the request" }, + { + kind: "text", + text: "**Reading the request**\nMock reasoning for call 1.", + }, + { kind: "done", text: "" }, + ]); + expect(first.reasoning).toEqual({ + summary: ["**Reading the request**\nMock reasoning for call 1."], + tokens: 8, + }); + expect(first.providerState).toEqual({ + providerType: "codex", + model: "mock-model", + items: [{ type: "mock-reasoning", call: 1 }], + }); + }); + + it("stays silent without a request, and reports nothing for off", async () => { + const mock = new MockAIProvider({ responses: "a" }); + const onReasoning = vi.fn(); + const plain = await mock.generateCompletion(params({ onReasoning })); + expect(onReasoning).not.toHaveBeenCalled(); + expect(plain.reasoning).toBeUndefined(); + const off = await mock.generateCompletion( + params({ reasoning: { effort: "off", summaries: true }, onReasoning }), + ); + expect(off.reasoning).toEqual({ summary: [], tokens: null }); + expect(off.providerState).toBeUndefined(); + }); +}); + +describe("Chain runner reasoning pass-through", () => { + it("hands the same reasoning request to the fallback hop and counts it", async () => { + const failing = new MockAIProvider({ + providerType: "anthropic", + rejectWith: Object.assign(new Error("down"), { httpStatus: 503 }), + }); + const working = new MockAIProvider({ + providerType: "codex", + responses: "ok", + }); + const events: AiReasoningEvent[] = []; + const onReasoning = (e: AiReasoningEvent) => events.push(e); + const reasoning = { effort: "high" as const, summaries: true }; + + const out = await runRawCompletionWithFallback({ + userId: "u1", + providers: [ + { providerType: "anthropic", instance: failing }, + { providerType: "codex", instance: working }, + ], + params: params({ reasoning, onReasoning }), + surface: "coach", + ledger: createInMemoryProviderHealthLedger(), + }); + + expect(failing.calls[0].reasoning).toBe(reasoning); + expect(working.calls[0].reasoning).toBe(reasoning); + expect(working.calls[0].onReasoning).toBe(onReasoning); + expect(events.map((e) => e.kind)).toEqual(["title", "text", "done"]); + expect(out.result.providerState?.items).toHaveLength(1); + + const success = annotations.find((m) => "ai_reasoning_spend" in m); + expect(success).toMatchObject({ + ai_reasoning_spend: 8, + ai_reasoning_summary_count: 1, + }); + // Counts only: no model text, no provider state on the wide event. + const flat = JSON.stringify(annotations); + expect(flat).not.toContain("mock-reasoning"); + expect(flat).not.toContain("Reading the request"); + }); +}); diff --git a/src/lib/ai/mock-client.ts b/src/lib/ai/mock-client.ts index 6cb7a2b65..1b596662b 100644 --- a/src/lib/ai/mock-client.ts +++ b/src/lib/ai/mock-client.ts @@ -4,6 +4,7 @@ import type { CompletionParams, CompletionResult, } from "./types"; +import { reasoningTitleOf } from "./reasoning/support"; /** * Deterministic mock AI provider for tests. @@ -52,6 +53,14 @@ export interface MockAIProviderOptions { finishReason?: CompletionResult["finishReason"]; /** v1.20.0 — cachedInputTokens echoed on the result. Defaults to null. */ cachedInputTokens?: number | null; + /** + * v1.41 — what a call that asked for reasoning reports. Unset: a fixed + * summary per call (`**Reading the request**` + the call number), 8 tokens, + * and one `providerState` item naming the call. `false`: the mock never + * reasons, like a provider without a reasoning stream. Calls without + * `params.reasoning` (or with `off`) never get either, as on a real client. + */ + reasoning?: { summary: string[]; tokens: number | null } | false; } /** Type guard: a per-call array of tool-call sets. */ @@ -82,6 +91,7 @@ export class MockAIProvider implements AIProvider { private readonly cachedInputTokens: number | null; private readonly model: string; private readonly rejectWith: Error | undefined; + private readonly reasoning: MockAIProviderOptions["reasoning"]; private callIdx = 0; constructor(opts: MockAIProviderOptions = {}) { @@ -90,6 +100,7 @@ export class MockAIProvider implements AIProvider { this.rejectWith = opts.rejectWith; this.finishReason = opts.finishReason; this.cachedInputTokens = opts.cachedInputTokens ?? null; + this.reasoning = opts.reasoning; if (isToolCallMatrix(opts.toolCalls)) { this.toolCalls = opts.toolCalls; } else if (Array.isArray(opts.toolCalls)) { @@ -127,6 +138,7 @@ export class MockAIProvider implements AIProvider { ? this.toolCalls[Math.min(this.callIdx, this.toolCalls.length - 1)] : undefined; this.callIdx++; + const reasoned = this.reason(params, this.callIdx); return { content: this.responses[idx], tokensUsed: this.tokens[tokIdx] ?? null, @@ -135,6 +147,41 @@ export class MockAIProvider implements AIProvider { providerType: this.type, ...(toolCalls ? { toolCalls } : {}), finishReason: this.finishReason, + ...reasoned, + }; + } + + /** + * v1.41 — deterministic reasoning for a call that asked for it: the same + * events a real client reports (title, text, done) and a `providerState` + * the next round can be checked for. + */ + private reason( + params: CompletionParams, + call: number, + ): Pick { + if (!params.reasoning) return {}; + if (params.reasoning.effort === "off" || this.reasoning === false) { + params.onReasoning?.({ kind: "done", text: "" }); + return { reasoning: { summary: [], tokens: null } }; + } + const reasoning = this.reasoning ?? { + summary: [`**Reading the request**\nMock reasoning for call ${call}.`], + tokens: 8, + }; + for (const text of reasoning.summary) { + const title = reasoningTitleOf(text); + if (title) params.onReasoning?.({ kind: "title", text: title }); + params.onReasoning?.({ kind: "text", text }); + } + params.onReasoning?.({ kind: "done", text: "" }); + return { + reasoning: { summary: [...reasoning.summary], tokens: reasoning.tokens }, + providerState: { + providerType: this.type, + model: this.model, + items: [{ type: "mock-reasoning", call }], + }, }; } diff --git a/src/lib/ai/provider-runner.ts b/src/lib/ai/provider-runner.ts index 7bf914106..860da4973 100644 --- a/src/lib/ai/provider-runner.ts +++ b/src/lib/ai/provider-runner.ts @@ -413,6 +413,10 @@ async function runRawChain( surface: BudgetSurface; ledger?: ProviderHealthLedger; }, + // v1.41 — `params` reaches every candidate unchanged, `reasoning` and + // `onReasoning` included. A fallback hop may see `providerState` another + // provider made on an earlier round; each client re-inserts only its own + // and drops the rest, so nothing is filtered here. invoke: (candidate: ProviderChainResolved) => Promise, successMeta?: ( candidate: ProviderChainResolved, @@ -494,6 +498,16 @@ async function runRawChain( ai_chain_working_provider: candidate.providerType, ai_chain_fallback_count: hops.length, ...(successMeta ? successMeta(candidate) : {}), + // v1.41 — counts only. The summaries are model text and the + // provider state is opaque; neither ever reaches a wide event. The + // reasoning-token count is `spend`: a key naming tokens is blanked + // by the central redactor. + ...(result.reasoning + ? { + ai_reasoning_spend: result.reasoning.tokens, + ai_reasoning_summary_count: result.reasoning.summary.length, + } + : {}), }, }); return { From 6112cfc6d5f9422bf6c416f4d5621d9dfbc18fb7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Tue, 6 Oct 2026 19:26:30 +0200 Subject: [PATCH 7/7] test(ai): keep provider state out of logs and storage providerState may be read only in the clients, the contract and the Coach tool loop, never on a line that logs, annotates or persists, and has no schema column. --- .../provider-state-never-persisted.test.ts | 117 ++++++++++++++++++ 1 file changed, 117 insertions(+) create mode 100644 src/lib/ai/__tests__/provider-state-never-persisted.test.ts diff --git a/src/lib/ai/__tests__/provider-state-never-persisted.test.ts b/src/lib/ai/__tests__/provider-state-never-persisted.test.ts new file mode 100644 index 000000000..87f351dba --- /dev/null +++ b/src/lib/ai/__tests__/provider-state-never-persisted.test.ts @@ -0,0 +1,117 @@ +import { readdirSync, readFileSync, statSync } from "node:fs"; +import { join, relative } from "node:path"; +import { describe, expect, it } from "vitest"; + +/** + * v1.41 — `providerState` (encrypted reasoning items, signed thinking blocks, + * gateway reasoning details) lives in memory for one turn. It is opaque + * provider output tied to the request that produced it: never persisted, + * never logged, never passed to `annotate()`. + * + * The guard freezes where the name may appear at all — the clients that make + * and consume it, the contract, and the Coach tool loop that threads it from + * one round to the next — and, inside those files, forbids any line that + * names it alongside a logging or persistence call. A new reader elsewhere + * (a persistence helper, a route, a job, a log line) fails here and has to + * argue its case in this list. + */ +const ROOT = join(__dirname, "../../../.."); +const SRC = join(ROOT, "src"); + +const ALLOWED_FILES = new Set([ + "src/lib/ai/types.ts", + "src/lib/ai/codex-client.ts", + "src/lib/ai/openai-client.ts", + "src/lib/ai/anthropic-client.ts", + "src/lib/ai/local-client.ts", + "src/lib/ai/mock-client.ts", +]); +/** The Coach tool loop carries it between rounds of one turn. */ +const ALLOWED_PREFIXES = ["src/lib/ai/coach/tools/"]; + +/** + * A read (`x.providerState`) or a key / field declaration + * (`providerState: …`, `providerState?: …`). Comment lines are skipped. A + * local variable that merely shares the name (the onboarding screen's + * `const providerState = useAiProviderState()`) is a different thing and + * does not match. + */ +const STATE_REFERENCE = /\.providerState\b|\bproviderState\??\s*:/; + +const FORBIDDEN_ON_SAME_LINE = + /annotate\(|prisma\.|auditLog|console\.|logger\.|reportToGlitchtip|captureException|persist|JSON\.stringify|encrypt\(/; + +function walk(dir: string, out: string[] = []): string[] { + for (const name of readdirSync(dir)) { + if ( + name === "generated" || + name === "node_modules" || + name === "__tests__" + ) { + continue; + } + const full = join(dir, name); + if (statSync(full).isDirectory()) { + walk(full, out); + } else if (/\.(ts|tsx)$/.test(name) && !/\.test\.tsx?$/.test(name)) { + out.push(full); + } + } + return out; +} + +function mentions(): Array<{ file: string; lines: string[] }> { + return walk(SRC) + .map((full) => { + const lines = readFileSync(full, "utf8") + .split("\n") + .filter((l) => !/^\s*(\/\/|\*|\/\*)/.test(l)) + .filter((l) => STATE_REFERENCE.test(l)); + return { file: relative(ROOT, full).split("\\").join("/"), lines }; + }) + .filter((m) => m.lines.length > 0); +} + +describe("providerState never leaves the turn", () => { + const found = mentions(); + + it("finds the clients that make it (a guard that matches nothing proves nothing)", () => { + const files = found.map((m) => m.file); + // The contract, the three clients that run tool rounds (Codex, OpenAI, + // Anthropic) and the mock, today. Local never runs tools. + expect(files.length).toBeGreaterThanOrEqual(5); + for (const client of [ + "src/lib/ai/codex-client.ts", + "src/lib/ai/anthropic-client.ts", + "src/lib/ai/openai-client.ts", + "src/lib/ai/types.ts", + ]) { + expect(files).toContain(client); + } + }); + + it("appears only in the clients, the contract and the tool loop", () => { + const outside = found + .map((m) => m.file) + .filter( + (f) => + !ALLOWED_FILES.has(f) && + !ALLOWED_PREFIXES.some((p) => f.startsWith(p)), + ); + expect(outside).toEqual([]); + }); + + it("is never named on a line that logs, annotates or persists", () => { + const offending = found.flatMap((m) => + m.lines + .filter((l) => FORBIDDEN_ON_SAME_LINE.test(l)) + .map((l) => `${m.file}: ${l.trim()}`), + ); + expect(offending).toEqual([]); + }); + + it("has no column in the schema", () => { + const schema = readFileSync(join(ROOT, "prisma/schema.prisma"), "utf8"); + expect(schema).not.toMatch(/providerState|provider_state/); + }); +});