diff --git a/src/lib/ai/__tests__/fixtures/reasoning/gateway-reasoning-rejected.json b/src/lib/ai/__tests__/fixtures/reasoning/gateway-reasoning-rejected.json new file mode 100644 index 000000000..1157ee15d --- /dev/null +++ b/src/lib/ai/__tests__/fixtures/reasoning/gateway-reasoning-rejected.json @@ -0,0 +1,8 @@ +{ + "error": { + "message": "Unrecognized request argument supplied: reasoning_effort", + "type": "invalid_request_error", + "param": null, + "code": null + } +} diff --git a/src/lib/ai/__tests__/fixtures/reasoning/openrouter-claude-after-tool.json b/src/lib/ai/__tests__/fixtures/reasoning/openrouter-claude-after-tool.json new file mode 100644 index 000000000..12698c190 --- /dev/null +++ b/src/lib/ai/__tests__/fixtures/reasoning/openrouter-claude-after-tool.json @@ -0,0 +1,36 @@ +{ + "id": "gen-fixture", + "object": "chat.completion", + "model": "anthropic/claude-haiku-4.5", + "service_tier": "default", + "choices": [ + { + "index": 0, + "logprobs": null, + "finish_reason": "stop", + "native_finish_reason": "end_turn", + "message": { + "role": "assistant", + "content": "Paris (the capital of France) currently has cloudy skies and a temperature of 18°C.", + "refusal": null, + "reasoning": null + } + } + ], + "usage": { + "prompt_tokens": 746, + "completion_tokens": 25, + "total_tokens": 771, + "prompt_tokens_details": { + "cached_tokens": 0, + "cache_write_tokens": 0, + "audio_tokens": 0, + "video_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0, + "image_tokens": 0, + "audio_tokens": 0 + } + } +} diff --git a/src/lib/ai/__tests__/fixtures/reasoning/openrouter-claude-tool-round.json b/src/lib/ai/__tests__/fixtures/reasoning/openrouter-claude-tool-round.json new file mode 100644 index 000000000..2ea53b8d8 --- /dev/null +++ b/src/lib/ai/__tests__/fixtures/reasoning/openrouter-claude-tool-round.json @@ -0,0 +1,56 @@ +{ + "id": "gen-fixture", + "object": "chat.completion", + "model": "anthropic/claude-haiku-4.5", + "service_tier": "default", + "choices": [ + { + "index": 0, + "logprobs": null, + "finish_reason": "tool_calls", + "native_finish_reason": "tool_use", + "message": { + "role": "assistant", + "content": null, + "refusal": null, + "reasoning": "The user is asking me to:\n1. Think about which city is the capital of France\n2. Call get_weather for that city\n\nThe capital of France is Paris. So I should call get_weather with \"Paris\" as the city parameter.", + "tool_calls": [ + { + "type": "function", + "index": 0, + "id": "toolu_bdrk_014bbCR7ZtuZso1nhAXhWT8u", + "function": { + "name": "get_weather", + "arguments": "{\"city\": \"Paris\"}" + } + } + ], + "reasoning_details": [ + { + "type": "reasoning.text", + "text": "The user is asking me to:\n1. Think about which city is the capital of France\n2. Call get_weather for that city\n\nThe capital of France is Paris. So I should call get_weather with \"Paris\" as the city parameter.", + "format": "anthropic-claude-v1", + "index": 0, + "signature": "sig-fixture-1" + } + ] + } + } + ], + "usage": { + "prompt_tokens": 609, + "completion_tokens": 114, + "total_tokens": 723, + "prompt_tokens_details": { + "cached_tokens": 0, + "cache_write_tokens": 0, + "audio_tokens": 0, + "video_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 58, + "image_tokens": 0, + "audio_tokens": 0 + } + } +} diff --git a/src/lib/ai/__tests__/local-reasoning.test.ts b/src/lib/ai/__tests__/local-reasoning.test.ts new file mode 100644 index 000000000..5c0a27c65 --- /dev/null +++ b/src/lib/ai/__tests__/local-reasoning.test.ts @@ -0,0 +1,278 @@ +import { readFileSync } from "node:fs"; +import { join } from "node:path"; +import { beforeEach, describe, expect, it, vi } from "vitest"; + +// Destination policy has its own suites; stub it so the wire can be asserted +// against the global fetch stub. +vi.mock("../local-host-allowlist", () => ({ aiEgressPolicyFor: () => ({}) })); +import { + LocalOpenAICompatibleClient, + resetLocalJsonDialectCache, +} from "../local-client"; +import { resetReasoningDialectCache } from "../reasoning/dialect-cache"; +import type { AiReasoningEvent, CompletionParams } from "../types"; + +/** + * v1.41 — reasoning on local OpenAI-compatible servers. The reply shapes are + * the documented ones: `reasoning_content` (llama.cpp, LiteLLM), `reasoning` + * (vLLM, Ollama, LM Studio), on the message and on each streamed delta, and + * an inline leading `` block. + */ +const FIXTURES = join(__dirname, "fixtures/reasoning"); + +const BASE = "http://localhost:11434/v1"; + +function buffered(message: Record) { + return async () => + new Response( + JSON.stringify({ + choices: [{ message, finish_reason: "stop" }], + usage: { + total_tokens: 70, + completion_tokens_details: { reasoning_tokens: 30 }, + }, + }), + { status: 200, headers: { "content-type": "application/json" } }, + ); +} + +function stream(deltas: Array>) { + return async () => { + const encoder = new TextEncoder(); + const frames = [ + ...deltas.map( + (delta) => + `data: ${JSON.stringify({ choices: [{ delta, finish_reason: null }] })}\n\n`, + ), + `data: ${JSON.stringify({ choices: [{ delta: {}, finish_reason: "stop" }], usage: { total_tokens: 55 } })}\n\n`, + "data: [DONE]\n\n", + ]; + return new Response( + new ReadableStream({ + start(c) { + for (const f of frames) c.enqueue(encoder.encode(f)); + c.close(); + }, + }), + { status: 200, headers: { "content-type": "text/event-stream" } }, + ); + }; +} + +function params(extra: Partial = {}): CompletionParams { + return { + system: "s", + messages: [{ role: "user", content: "u" }], + maxTokens: 600, + ...extra, + }; +} + +const body = (mock: ReturnType, call = 0) => + JSON.parse(mock.mock.calls[call][1].body); + +const client = () => + new LocalOpenAICompatibleClient({ model: "qwen3:8b", baseUrl: BASE }); + +beforeEach(() => { + vi.restoreAllMocks(); + resetLocalJsonDialectCache(); + resetReasoningDialectCache(); +}); + +describe("Local reasoning request", () => { + it("sends the call's level, off as none, and gives it room to think", async () => { + const mock = vi.fn().mockImplementation(buffered({ content: "ok" })); + vi.stubGlobal("fetch", mock); + const c = client(); + c.reasoningEffort = "high"; + await c.generateCompletion(params()); + await c.generateCompletion( + params({ reasoning: { effort: "medium", summaries: true } }), + ); + await c.generateCompletion( + params({ reasoning: { effort: "off", summaries: true } }), + ); + // No level on the call: the entry's own setting, as before. + expect(body(mock, 0).reasoning_effort).toBe("high"); + expect(body(mock, 0).max_tokens).toBe(600); + expect(body(mock, 1).reasoning_effort).toBe("medium"); + expect(body(mock, 1).max_tokens).toBe(600 + 4096); + expect(body(mock, 2).reasoning_effort).toBe("none"); + expect(body(mock, 2).max_tokens).toBe(600); + }); + + it("retries once without the field when the server refuses it, and remembers", async () => { + const refusal = readFileSync( + join(FIXTURES, "gateway-reasoning-rejected.json"), + "utf8", + ); + const mock = vi + .fn() + .mockImplementationOnce( + async () => new Response(refusal, { status: 400 }), + ) + .mockImplementation(buffered({ content: "ok" })); + vi.stubGlobal("fetch", mock); + const c = client(); + const result = await c.generateCompletion( + params({ reasoning: { effort: "low", summaries: true } }), + ); + expect(result.content).toBe("ok"); + expect(body(mock, 1)).not.toHaveProperty("reasoning_effort"); + await c.generateCompletion( + params({ reasoning: { effort: "low", summaries: true } }), + ); + expect(mock).toHaveBeenCalledTimes(3); + expect(body(mock, 2)).not.toHaveProperty("reasoning_effort"); + }); +}); + +describe("Local reasoning replies", () => { + it.each([["reasoning_content"], ["reasoning"]])( + "reads %s off the message", + async (field) => { + vi.stubGlobal( + "fetch", + vi + .fn() + .mockImplementation( + buffered({ content: "Answer.", [field]: "**Plan**\nThink." }), + ), + ); + const events: AiReasoningEvent[] = []; + const result = await client().generateCompletion( + params({ + reasoning: { effort: "low", summaries: true }, + onReasoning: (e) => events.push(e), + }), + ); + expect(result.content).toBe("Answer."); + expect(result.reasoning).toEqual({ + summary: ["**Plan**\nThink."], + tokens: 30, + }); + expect(events).toEqual([ + { kind: "title", text: "Plan" }, + { kind: "text", text: "**Plan**\nThink." }, + { kind: "done", text: "" }, + ]); + }, + ); + + it("splits an inline think block only for a caller that asked for reasoning", async () => { + const reply = "check the window\n\nThe answer."; + vi.stubGlobal( + "fetch", + vi.fn().mockImplementation(buffered({ content: reply })), + ); + const asked = await client().generateCompletion( + params({ reasoning: { effort: "low", summaries: true } }), + ); + expect(asked.content).toBe("The answer."); + expect(asked.reasoning?.summary).toEqual(["check the window"]); + const unasked = await client().generateCompletion(params()); + expect(unasked.content).toBe(reply); + expect(unasked.reasoning).toBeUndefined(); + }); +}); + +describe("Local reasoning stream", () => { + it("reports streamed reasoning live and keeps it out of the answer", async () => { + vi.stubGlobal( + "fetch", + vi + .fn() + .mockImplementation( + stream([ + { reasoning_content: "**Choosing" }, + { reasoning_content: " the window**\n" }, + { reasoning_content: "Ninety days." }, + { content: "Your " }, + { content: "answer." }, + ]), + ), + ); + const events: AiReasoningEvent[] = []; + const deltas: string[] = []; + const result = await client().generateCompletionStream( + params({ + reasoning: { effort: "medium", summaries: true }, + onReasoning: (e) => events.push(e), + }), + (d) => deltas.push(d), + ); + expect(deltas).toEqual(["Your ", "answer."]); + expect(result.content).toBe("Your answer."); + expect(events).toEqual([ + { kind: "title", text: "Choosing the window" }, + { kind: "text", text: "**Choosing the window**\nNinety days." }, + { kind: "done", text: "" }, + ]); + expect(result.reasoning?.summary).toEqual([ + "**Choosing the window**\nNinety days.", + ]); + }); + + it("routes an inline think block split across chunks to the reasoning", async () => { + vi.stubGlobal( + "fetch", + vi + .fn() + .mockImplementation( + stream([ + { content: " weigh the" }, + { content: " two readings\n\nThe " }, + { content: "answer." }, + ]), + ), + ); + const deltas: string[] = []; + const result = await client().generateCompletionStream( + params({ reasoning: { effort: "low", summaries: true } }), + (d) => deltas.push(d), + ); + expect(deltas.join("")).toBe("The answer."); + expect(result.content).toBe("The answer."); + expect(result.reasoning?.summary).toEqual(["weigh the two readings"]); + }); + + it("streams untouched for a caller that asked for no reasoning", async () => { + vi.stubGlobal( + "fetch", + vi + .fn() + .mockImplementation( + stream([ + { reasoning: "ignored" }, + { content: "x" }, + { content: "y" }, + ]), + ), + ); + const deltas: string[] = []; + const onReasoning = vi.fn(); + const result = await client().generateCompletionStream( + params({ onReasoning }), + (d) => deltas.push(d), + ); + expect(deltas).toEqual(["x", "y"]); + expect(result.content).toBe("xy"); + expect(result.reasoning).toBeUndefined(); + expect(onReasoning).not.toHaveBeenCalled(); + }); + + it("passes a short reply that only looked like it might open a think block", async () => { + vi.stubGlobal( + "fetch", + vi.fn().mockImplementation(stream([{ content: " {}, + ); + expect(result.content).toBe(" { + const actual = await importOriginal(); + return { + ...actual, + fetch: (input: unknown, init?: unknown) => + (globalThis.fetch as unknown as (i: unknown, n?: unknown) => unknown)( + input, + init, + ), + }; +}); + +/** + * v1.41 — reasoning on the Chat Completions wire. The OpenRouter fixtures + * were recorded on 2026-10-06 with a throwaway prompt (no health data); + * signatures and encrypted payloads are placeholders. The gateway refusal is + * the standard OpenAI error envelope. + */ +const FIXTURES = join(__dirname, "fixtures/reasoning"); +const fixture = (name: string) => readFileSync(join(FIXTURES, name), "utf8"); +const ok = (name: string) => async () => + new Response(fixture(name), { + status: 200, + headers: { "content-type": "application/json" }, + }); +const plain = + (content: string, extra: Record = {}) => + async () => + new Response( + JSON.stringify({ + choices: [ + { + message: { role: "assistant", content, ...extra }, + finish_reason: "stop", + }, + ], + usage: { + total_tokens: 90, + completion_tokens_details: { reasoning_tokens: 40 }, + }, + }), + { status: 200 }, + ); + +const TOOLS: AiToolDef[] = [ + { + name: "get_weather", + description: "Weather for a city", + parameters: { type: "object", properties: { city: { type: "string" } } }, + }, +]; + +function params(extra: Partial = {}): CompletionParams { + return { + system: "You are terse.", + messages: [{ role: "user", content: "Weather in the capital of France?" }], + maxTokens: 600, + ...extra, + }; +} + +const body = (mock: ReturnType, call = 0) => + JSON.parse(mock.mock.calls[call][1].body); + +const openai = (model: string) => + new OpenAIClient({ + apiKey: "sk-test", + model, + baseUrl: "https://api.openai.com/v1", + }); +const gateway = (baseUrl: string, model: string) => + new OpenAIClient({ + apiKey: "sk-test", + model, + baseUrl, + providerType: "openai-compatible", + }); + +beforeEach(() => { + vi.restoreAllMocks(); + resetReasoningDialectCache(); + resetJsonModeDialectCache(); +}); + +describe("OpenAI reasoning", () => { + it("sends nothing new when the call asks for no reasoning", async () => { + const mock = vi.fn().mockImplementation(plain("ok")); + vi.stubGlobal("fetch", mock); + const result = await openai("gpt-5.5").generateCompletion(params()); + expect(body(mock)).not.toHaveProperty("reasoning_effort"); + expect(body(mock).max_completion_tokens).toBe(600); + expect(result.reasoning).toBeUndefined(); + }); + + it("sends the effort on a round without tools, with room to think", async () => { + const mock = vi.fn().mockImplementation(plain("ok")); + vi.stubGlobal("fetch", mock); + const result = await openai("gpt-5.5").generateCompletion( + params({ reasoning: { effort: "medium", summaries: true } }), + ); + expect(body(mock).reasoning_effort).toBe("medium"); + expect(body(mock).max_completion_tokens).toBe(600 + 4096); + expect(body(mock)).not.toHaveProperty("temperature"); + expect(result.reasoning).toEqual({ summary: [], tokens: 40 }); + }); + + it("sends no effort on a tool round", async () => { + const mock = vi.fn().mockImplementation(plain("ok")); + vi.stubGlobal("fetch", mock); + await openai("gpt-5.5").generateCompletion( + params({ tools: TOOLS, reasoning: { effort: "high", summaries: true } }), + ); + expect(body(mock)).not.toHaveProperty("reasoning_effort"); + expect(body(mock).max_completion_tokens).toBe(600); + }); + + it("maps off onto each family's lowest effort, and skips non-reasoning models", async () => { + const mock = vi.fn().mockImplementation(plain("ok")); + vi.stubGlobal("fetch", mock); + const off = { reasoning: { effort: "off" as const, summaries: false } }; + await openai("gpt-5.5").generateCompletion(params(off)); + await openai("gpt-5-mini").generateCompletion(params(off)); + await openai("o3").generateCompletion(params(off)); + await openai("gpt-4o").generateCompletion(params(off)); + expect(body(mock, 0).reasoning_effort).toBe("none"); + expect(body(mock, 1).reasoning_effort).toBe("minimal"); + expect(body(mock, 2).reasoning_effort).toBe("low"); + expect(body(mock, 3)).not.toHaveProperty("reasoning_effort"); + }); +}); + +describe("Gateway reasoning", () => { + it("lets the call's level win over the entry's stored setting", async () => { + const mock = vi.fn().mockImplementation(plain("ok")); + vi.stubGlobal("fetch", mock); + const c = gateway("https://litellm.example.com/v1", "qwen3"); + c.reasoningEffort = "high"; + await c.generateCompletion(params()); + await c.generateCompletion( + params({ reasoning: { effort: "low", summaries: true } }), + ); + expect(body(mock, 0).reasoning_effort).toBe("high"); + expect(body(mock, 1).reasoning_effort).toBe("low"); + }); + + it("reads reasoning_content as the summary", async () => { + vi.stubGlobal( + "fetch", + vi + .fn() + .mockImplementation( + plain("Paris.", { reasoning_content: "**Capital**\nIt is Paris." }), + ), + ); + const events: AiReasoningEvent[] = []; + const result = await gateway( + "https://litellm.example.com/v1", + "qwen3", + ).generateCompletion( + params({ + reasoning: { effort: "medium", summaries: true }, + onReasoning: (e) => events.push(e), + }), + ); + expect(result.content).toBe("Paris."); + expect(result.reasoning?.summary).toEqual(["**Capital**\nIt is Paris."]); + expect(events.map((e) => e.kind)).toEqual(["title", "text", "done"]); + }); + + it("speaks OpenRouter's dialect and carries reasoning_details to the next round", async () => { + const mock = vi + .fn() + .mockImplementationOnce(ok("openrouter-claude-tool-round.json")) + .mockImplementation(ok("openrouter-claude-after-tool.json")); + vi.stubGlobal("fetch", mock); + const c = gateway( + "https://openrouter.ai/api/v1", + "anthropic/claude-haiku-4.5", + ); + const first = await c.generateCompletion( + params({ tools: TOOLS, reasoning: { effort: "low", summaries: true } }), + ); + expect(body(mock, 0).reasoning).toEqual({ effort: "low" }); + expect(body(mock, 0)).not.toHaveProperty("reasoning_effort"); + expect(first.finishReason).toBe("tool_calls"); + expect(first.reasoning?.tokens).toBe(58); + expect(first.reasoning?.summary[0]).toMatch(/capital of France is Paris/); + const details = JSON.parse(fixture("openrouter-claude-tool-round.json")) + .choices[0].message.reasoning_details; + expect(first.providerState).toEqual({ + providerType: "openai-compatible", + model: "anthropic/claude-haiku-4.5", + items: [{ reasoning_details: details }], + }); + + await c.generateCompletion( + params({ + tools: TOOLS, + reasoning: { effort: "low", summaries: true }, + messages: [ + { role: "user", content: "Weather in the capital of France?" }, + { + role: "assistant", + content: first.content, + toolCalls: first.toolCalls, + providerState: first.providerState, + }, + { role: "tool", toolCallId: first.toolCalls![0].id, content: "{}" }, + ], + }), + ); + const assistant = body(mock, 1).messages[2]; + expect(assistant.role).toBe("assistant"); + expect(assistant.reasoning_details).toEqual(details); + }); + + it("does not hand state to a different model", async () => { + const mock = vi.fn().mockImplementation(plain("ok")); + vi.stubGlobal("fetch", mock); + await gateway( + "https://openrouter.ai/api/v1", + "openai/gpt-5-mini", + ).generateCompletion( + params({ + messages: [ + { role: "user", content: "q" }, + { + role: "assistant", + content: "", + toolCalls: [{ id: "c1", name: "get_weather", arguments: "{}" }], + providerState: { + providerType: "openai-compatible", + model: "anthropic/claude-haiku-4.5", + items: [{ reasoning_details: [{ type: "reasoning.text" }] }], + }, + }, + { role: "tool", toolCallId: "c1", content: "{}" }, + ], + }), + ); + expect(body(mock).messages[2]).not.toHaveProperty("reasoning_details"); + }); + + it("retries once without reasoning when the gateway refuses it, and remembers", async () => { + const mock = vi + .fn() + .mockImplementationOnce( + async () => + new Response(fixture("gateway-reasoning-rejected.json"), { + status: 400, + }), + ) + .mockImplementation(plain("ok")); + vi.stubGlobal("fetch", mock); + const c = gateway("https://litellm.example.com/v1", "llama3"); + const result = await c.generateCompletion( + params({ + responseFormat: "json", + reasoning: { effort: "medium", summaries: true }, + }), + ); + expect(body(mock, 0).reasoning_effort).toBe("medium"); + expect(body(mock, 1)).not.toHaveProperty("reasoning_effort"); + // The JSON flag survives: the refusal named the reasoning field. + expect(body(mock, 1).response_format).toEqual({ type: "json_object" }); + expect(jsonModeDialectFor("https://litellm.example.com/v1")).toBe( + "response_format", + ); + expect(result.reasoning?.downgradedTo).toBe("off"); + + await c.generateCompletion( + params({ reasoning: { effort: "medium", summaries: true } }), + ); + expect(mock).toHaveBeenCalledTimes(3); + expect(body(mock, 2)).not.toHaveProperty("reasoning_effort"); + }); +}); diff --git a/src/lib/ai/local-client.ts b/src/lib/ai/local-client.ts index e753050ef..2f6ef99e6 100644 --- a/src/lib/ai/local-client.ts +++ b/src/lib/ai/local-client.ts @@ -24,6 +24,13 @@ import { type JsonModeDialect, } from "./json-dialect"; import type { ReasoningEffort } from "./reasoning-effort"; +import { + annotateReasoningDowngrade, + learnedReasoningDialect, + rememberReasoningDialect, +} from "./reasoning/dialect-cache"; +import { REASONING_THINKING_BUDGET } from "./reasoning/levels"; +import { reasoningTitleOf, splitThinkTags } from "./reasoning/support"; interface LocalClientConfig { apiKey?: string | null; @@ -88,6 +95,68 @@ function withStrictJsonPrefix(messages: AiMessage[]): AiMessage[] { return out; } +/** + * v1.41 — collects what a call reasoned and reports it to `onReasoning`. + * Local servers return the reasoning as `reasoning_content` (llama.cpp, + * LiteLLM) or `reasoning` (vLLM, Ollama, LM Studio), on the message and on + * each streamed delta, or inline as a leading `…` block. + * https://github.com/ggml-org/llama.cpp/blob/master/tools/server/README.md, + * https://docs.vllm.ai/en/latest/features/reasoning_outputs.html, + * https://docs.ollama.com/api/openai-compatibility + */ +class ReasoningCollector { + private text = ""; + private titled = false; + private finished = false; + constructor(private readonly params: CompletionParams) {} + + private get emit() { + return this.params.reasoning ? this.params.onReasoning : undefined; + } + + /** A streamed piece of reasoning; the title goes out once it has closed. */ + add(delta: string): void { + this.text += delta; + if (!this.titled && this.emit) { + const title = reasoningTitleOf(this.text); + if (title) { + this.titled = true; + this.emit({ kind: "title", text: title }); + } + } + } + + /** The reasoning is complete (the answer started, or the call ended). */ + finish(): void { + if (this.finished) return; + this.finished = true; + const text = this.text.trim(); + if (!text || !this.emit) return; + if (!this.titled) { + const title = reasoningTitleOf(text); + if (title) this.emit({ kind: "title", text: title }); + } + this.emit({ kind: "text", text }); + } + + result(tokens: number | null): CompletionResult["reasoning"] { + this.finish(); + if (!this.params.reasoning) return undefined; + this.emit?.({ kind: "done", text: "" }); + const text = this.text.trim(); + return { summary: text ? [text] : [], tokens }; + } +} + +/** The reasoning text a non-streamed message carries, if any. */ +function messageReasoning(message: unknown): string | null { + const m = (message ?? {}) as Record; + const text = [m.reasoning_content, m.reasoning].find( + (v): v is string => typeof v === "string" && v.trim().length > 0, + ); + return text ?? null; +} + /** * Talks to OpenAI-compatible servers: local runtimes (Ollama, LocalAI, * LM Studio, vLLM, …) AND hosted gateways (LiteLLM, OpenRouter, any @@ -158,11 +227,18 @@ export class LocalOpenAICompatibleClient implements AIProvider { withStrictJsonPrefix(params.messages), ); + const callEffort = this.callReasoningEffort(params); + // Reasoning tokens count against `max_tokens` on these servers; the + // thinking budget rides on top of the answer budget (a ceiling only). + const thinkingTokens = + params.reasoning && callEffort && params.reasoning.effort !== "off" + ? REASONING_THINKING_BUDGET[params.reasoning.effort] + : 0; const body = JSON.stringify({ model: this.config.model, messages, temperature: params.temperature ?? 0.3, - max_tokens: params.maxTokens ?? 1000, + max_tokens: (params.maxTokens ?? 1000) + thinkingTokens, // Deterministic seed for reproducible reference output. Ollama + // most OpenAI-compatible local servers honour it; servers that // ignore it simply disregard the field. Omitted when unset. @@ -170,9 +246,11 @@ export class LocalOpenAICompatibleClient implements AIProvider { // #1126 — the entry's reasoning setting. Default sends nothing, so the // body of a setup that never touched it stays byte-identical; "Off" is // `none`, which stops a thinking model spending the answer budget. - ...(this.reasoningEffort + // v1.41 — a call that carries its own resolved level wins over it. + ...(!params.reasoning && this.reasoningEffort ? { reasoning_effort: this.reasoningEffort } : {}), + ...(callEffort ? { reasoning_effort: callEffort } : {}), // v1.28.28 (#470) — the JSON surfaces opt in via `responseFormat: // "json"`. Default dialect sends the STANDARD OpenAI field (the // top-level Ollama-native `format: "json"` this client used to send @@ -194,6 +272,27 @@ export class LocalOpenAICompatibleClient implements AIProvider { return { url, headers, body, jsonDialect }; } + private reasoningDialectKey() { + return { + provider: "local", + endpoint: this.config.baseUrl, + model: this.config.model, + }; + } + + /** + * The `reasoning_effort` a call with its own resolved level sends (`off` + * is `none`), unless this endpoint refused the field before. `null` for a + * call without one: the entry's #1126 setting applies, exactly as before. + */ + private callReasoningEffort(params: CompletionParams): string | null { + if (!params.reasoning) return null; + if (learnedReasoningDialect(this.reasoningDialectKey()) === "none") { + return null; + } + return params.reasoning.effort === "off" ? "none" : params.reasoning.effort; + } + async generateCompletion( params: CompletionParams, ): Promise { @@ -243,6 +342,20 @@ export class LocalOpenAICompatibleClient implements AIProvider { // strip anything that looks like a key before logging. const rawBody = await res.text().catch(() => ""); const bodyExcerpt = sanitiseBodyExcerpt(rawBody); + // v1.41 — a server that refuses the reasoning field is asked once more + // without it and remembers. Before the JSON dialect, so a complaint + // about reasoning never switches JSON mode off. + const sentEffort = this.callReasoningEffort(params); + if ( + sentEffort && + res.status >= 400 && + res.status < 500 && + /reasoning|think/i.test(bodyExcerpt) + ) { + rememberReasoningDialect(this.reasoningDialectKey(), "none"); + annotateReasoningDowngrade("local", sentEffort, "none"); + return this.generateCompletion(params); + } // v1.28.28 (#470) — dialect self-heal. A 4xx whose body names // `response_format` (or an unknown parameter) means this endpoint // rejects the standard JSON flag: learn the no-flag dialect for the @@ -292,7 +405,10 @@ export class LocalOpenAICompatibleClient implements AIProvider { message?: { content?: string }; finish_reason?: string; }>; - usage?: { total_tokens?: number }; + usage?: { + total_tokens?: number; + completion_tokens_details?: { reasoning_tokens?: number }; + }; }; // Same embedded-failure shape the OpenAI-compatible gateway tag handles: @@ -307,7 +423,22 @@ export class LocalOpenAICompatibleClient implements AIProvider { model: this.config.model, }); - const content = json.choices?.[0]?.message?.content; + const collector = new ReasoningCollector(params); + let content = json.choices?.[0]?.message?.content; + if (params.reasoning) { + const fielded = messageReasoning(json.choices?.[0]?.message); + if (fielded) collector.add(fielded); + // Only a caller that asked for reasoning gets the inline block split + // off; every other caller keeps the text exactly as it came. + if (content) { + const split = splitThinkTags(content); + if (split.reasoning) collector.add(split.reasoning); + content = split.content; + } + } + const reasoning = collector.result( + json.usage?.completion_tokens_details?.reasoning_tokens ?? null, + ); if (!content) { // v1.20.1 — sentinel httpStatus + kind so the chain classifier can tell // an empty 200-OK reply apart from a transport failure. Cascade unchanged @@ -329,6 +460,7 @@ export class LocalOpenAICompatibleClient implements AIProvider { // v1.28.28 (#470) — surface why the model stopped so the JSON callers // can tell a token-ceiling truncation ("length") from bad output. finishReason: mapFinishReason(json.choices?.[0]?.finish_reason), + ...(reasoning ? { reasoning } : {}), }; } @@ -416,7 +548,18 @@ export class LocalOpenAICompatibleClient implements AIProvider { label: "local AI endpoint", model: this.config.model, }); - const buffered = json?.choices?.[0]?.message?.content; + const collector = new ReasoningCollector(params); + let buffered = json?.choices?.[0]?.message?.content; + if (params.reasoning) { + const fielded = messageReasoning(json?.choices?.[0]?.message); + if (fielded) collector.add(fielded); + if (buffered) { + const split = splitThinkTags(buffered); + if (split.reasoning) collector.add(split.reasoning); + buffered = split.content; + } + } + const reasoning = collector.result(null); if (!buffered) { const err = new Error("Local AI returned empty content"); Object.assign(err, { @@ -433,6 +576,7 @@ export class LocalOpenAICompatibleClient implements AIProvider { model: this.config.model, providerType: "local", finishReason: mapFinishReason(json?.choices?.[0]?.finish_reason), + ...(reasoning ? { reasoning } : {}), }; } @@ -441,8 +585,55 @@ export class LocalOpenAICompatibleClient implements AIProvider { let sseBuffer = ""; let content = ""; let tokensUsed: number | null = null; + let reasoningTokens: number | null = null; let finishReason: string | undefined; + // v1.41 — reasoning on the stream: the `reasoning_content` / `reasoning` + // delta fields, or a leading `` block inside `content`. Inline + // reasoning never reaches `onDelta`; it would otherwise show up as the + // answer. Only for a caller that asked for reasoning. + const collector = new ReasoningCollector(params); + let thinkState: "detect" | "inside" | "passthrough" = params.reasoning + ? "detect" + : "passthrough"; + let pending = ""; + const passContent = (text: string) => { + if (!text) return; + collector.finish(); + content += text; + onDelta(text); + }; + const onContent = (delta: string) => { + if (thinkState === "passthrough") { + passContent(delta); + return; + } + pending += delta; + if (thinkState === "detect") { + const lead = pending.replace(/^\s+/, ""); + if (lead.length < "".length && "".startsWith(lead)) { + return; + } + if (!lead.startsWith("")) { + thinkState = "passthrough"; + const out = pending; + pending = ""; + passContent(out); + return; + } + thinkState = "inside"; + pending = lead.slice("".length); + } + const close = pending.indexOf(""); + if (close === -1) return; + collector.add(pending.slice(0, close)); + collector.finish(); + const rest = pending.slice(close + "".length).replace(/^\s+/, ""); + pending = ""; + thinkState = "passthrough"; + passContent(rest); + }; + // An SSE frame can carry the same embedded failure the buffered body // carries. Remember the first one; it only matters if the stream ends // without a single token, in which case it is the honest reason. @@ -474,11 +665,18 @@ export class LocalOpenAICompatibleClient implements AIProvider { let chunk: { error?: unknown; choices?: Array<{ - delta?: { content?: string }; + delta?: { + content?: string; + reasoning_content?: string; + reasoning?: string; + }; finish_reason?: string | null; error?: unknown; }>; - usage?: { total_tokens?: number }; + usage?: { + total_tokens?: number; + completion_tokens_details?: { reasoning_tokens?: number }; + }; }; try { chunk = JSON.parse(payload); @@ -489,14 +687,19 @@ export class LocalOpenAICompatibleClient implements AIProvider { if (streamedError === null && readEmbeddedError(chunk) !== null) { streamedError = chunk; } - const delta = chunk.choices?.[0]?.delta?.content; - if (delta) { - content += delta; - onDelta(delta); + const choiceDelta = chunk.choices?.[0]?.delta; + const reasoningDelta = + choiceDelta?.reasoning_content ?? choiceDelta?.reasoning; + if (params.reasoning && typeof reasoningDelta === "string") { + collector.add(reasoningDelta); } + const delta = choiceDelta?.content; + if (delta) onContent(delta); if (typeof chunk.usage?.total_tokens === "number") { tokensUsed = chunk.usage.total_tokens; } + const rt = chunk.usage?.completion_tokens_details?.reasoning_tokens; + if (typeof rt === "number") reasoningTokens = rt; const chunkFinish = chunk.choices?.[0]?.finish_reason; if (typeof chunkFinish === "string") finishReason = chunkFinish; } @@ -521,6 +724,19 @@ export class LocalOpenAICompatibleClient implements AIProvider { reader.cancel().catch(() => {}); } + // A reply short enough to still sit in the `` detector is prose; + // an unclosed block is reasoning that ran out of budget. + // (The closures above move `thinkState`; read it afresh.) + const endState = thinkState as "detect" | "inside" | "passthrough"; + if (endState === "detect") { + thinkState = "passthrough"; + passContent(pending); + } else if (endState === "inside") { + collector.add(pending); + } + pending = ""; + const reasoning = collector.result(reasoningTokens); + if (!content) { // A stream that ended with a failure frame and no token reports the // failure, not "empty content". @@ -544,6 +760,7 @@ export class LocalOpenAICompatibleClient implements AIProvider { model: this.config.model, providerType: "local", finishReason: mapFinishReason(finishReason), + ...(reasoning ? { reasoning } : {}), }; } } diff --git a/src/lib/ai/openai-client.ts b/src/lib/ai/openai-client.ts index f54ea1bf1..e8acd00bd 100644 --- a/src/lib/ai/openai-client.ts +++ b/src/lib/ai/openai-client.ts @@ -2,7 +2,23 @@ import { safeFetch } from "@/lib/safe-fetch"; import { callTimeoutMs } from "./effective-timeout"; import { annotate } from "@/lib/logging/context"; import type { AIProvider, CompletionParams, CompletionResult } from "./types"; -import { selectOpenAIChatCompletionsCapabilities } from "./openai-capabilities"; +import { + selectOpenAIChatCompletionsCapabilities, + type OpenAIChatCompletionsCapabilities, +} from "./openai-capabilities"; +import { + annotateReasoningDowngrade, + learnedReasoningDialect, + rememberReasoningDialect, +} from "./reasoning/dialect-cache"; +import { REASONING_THINKING_BUDGET } from "./reasoning/levels"; +import { + isOpenRouterEndpoint, + levelOfWireEffort, + openAIOffEffort, + openAIReasoningFamily, + reasoningTitleOf, +} from "./reasoning/support"; import { hasLearnedJsonModeDialect, isResponseFormatRejection, @@ -59,6 +75,26 @@ interface OpenAIClientConfig { type OpenAIProviderType = "admin-key" | "codex" | "openai-compatible"; +/** + * v1.41 — the reasoning one request sends. + * - `reasoning_effort`: the Chat Completions field (OpenAI, LiteLLM, vLLM, + * most gateways). https://platform.openai.com/docs/api-reference/chat/create + * - `openrouter`: OpenRouter's `reasoning: { effort }` object, which also + * returns `reasoning_details` to hand back on tool rounds. + * https://openrouter.ai/docs/use-cases/reasoning-tokens + */ +type OpenAIReasoningWire = + | { kind: "none" } + | { kind: "reasoning_effort"; effort: string } + | { kind: "openrouter"; effort: string }; + +/** + * Assistant-message fields a gateway returns and needs back unmodified on the + * next tool round: OpenRouter's `reasoning_details` and the `thinking_blocks` + * LiteLLM carries for Claude upstreams (https://docs.litellm.ai/docs/reasoning_content). + */ +const ROUND_TRIP_FIELDS = ["reasoning_details", "thinking_blocks"] as const; + export class OpenAIClient implements AIProvider { readonly type: OpenAIProviderType; /** See `AIProvider.responseTimeoutSeconds`; stamped by the resolver. */ @@ -94,7 +130,10 @@ export class OpenAIClient implements AIProvider { // conversation turns mapped 1:1. Vision parts (Lab-OCR) become the // multimodal `image_url` content array `gpt-4o`-class models accept. The // image is framed as untrusted DATA by the system prompt. - const messages = buildOpenAIMessages(params.system, params.messages); + const messages = this.withRoundTripState( + buildOpenAIMessages(params.system, params.messages), + params, + ); // v1.20.0 — tool plumbing. The defs map onto the OpenAI `function` tool // wire; F1 supplies real defs and consumes the parsed `toolCalls`. No F4 @@ -122,7 +161,19 @@ export class OpenAIClient implements AIProvider { this.config.baseUrl, this.config.model, ); - const tokenBudget = params.maxTokens ?? 1000; + const reasoningWire = this.planReasoning(params, hasTools, capabilities); + // Reasoning tokens count against the completion budget on every one of + // these wires, so the thinking budget rides on top of the answer budget + // (a ceiling, not a charge). Without it a 600-token Coach round spends + // its whole allowance thinking and returns no answer. + const wireLevel = + reasoningWire.kind === "none" || reasoningWire.effort === "none" + ? null + : levelOfWireEffort(reasoningWire.effort); + const thinkingTokens = wireLevel + ? REASONING_THINKING_BUDGET[wireLevel === "off" ? "low" : wireLevel] + : 0; + const tokenBudget = (params.maxTokens ?? 1000) + thinkingTokens; const capabilityParams = capabilities.supportsSamplingControls ? { [capabilities.tokenBudgetField]: tokenBudget, @@ -157,10 +208,16 @@ export class OpenAIClient implements AIProvider { ...(tools ? { tools } : {}), ...(params.toolChoice ? { tool_choice: params.toolChoice } : {}), // #1126 — the gateway entry's reasoning setting; Default sends - // nothing. `api.openai.com` and Codex never get it. - ...(this.isGateway && this.reasoningEffort + // nothing. `api.openai.com` and Codex never get it. v1.41 — a call + // that carries its own resolved level (`params.reasoning`) wins. + ...(!params.reasoning && this.isGateway && this.reasoningEffort ? { reasoning_effort: this.reasoningEffort } : {}), + ...(reasoningWire.kind === "reasoning_effort" + ? { reasoning_effort: reasoningWire.effort } + : reasoningWire.kind === "openrouter" + ? { reasoning: { effort: reasoningWire.effort } } + : {}), }), }, // 60 s ceiling so a tar-pit upstream cannot pin a worker @@ -205,6 +262,20 @@ export class OpenAIClient implements AIProvider { // that looks like an API key from the excerpt before logging. const rawBody = await res.text().catch(() => ""); const bodyExcerpt = sanitiseBodyExcerpt(rawBody); + // v1.41 — an endpoint or model that refuses the reasoning field is + // asked once more without it and remembers, so the answer never fails + // on a reasoning parameter. Checked before the JSON dialect: a body + // naming the reasoning field must not switch JSON mode off. + if ( + reasoningWire.kind !== "none" && + res.status >= 400 && + res.status < 500 && + /reasoning|thinking/i.test(bodyExcerpt) + ) { + rememberReasoningDialect(this.reasoningDialectKey(), "none"); + annotateReasoningDowngrade(this.type, reasoningWire.effort, "none"); + return this.generateCompletion(params); + } // v1.33.1 (#470) — dialect self-heal, gateway only. A 4xx whose body // names `response_format` (or an unknown parameter) means this endpoint // rejects the standard JSON flag: learn the no-flag dialect for the base @@ -320,6 +391,31 @@ export class OpenAIClient implements AIProvider { throw err; } + // v1.41 — what the call reasoned. Gateways return the text as + // `reasoning` (OpenRouter, vLLM, Ollama) or `reasoning_content` + // (LiteLLM, llama.cpp); `api.openai.com` returns none on this wire. + const message = (choice?.message ?? {}) as Record; + const reasoningText = [message.reasoning, message.reasoning_content].find( + (v): v is string => typeof v === "string" && v.trim().length > 0, + ); + const summary = reasoningText ? [reasoningText.trim()] : []; + const emit = params.reasoning ? params.onReasoning : undefined; + if (emit) { + for (const text of summary) { + const title = reasoningTitleOf(text); + if (title) emit({ kind: "title", text: title }); + emit({ kind: "text", text }); + } + emit({ kind: "done", text: "" }); + } + const roundTrip = ROUND_TRIP_FIELDS.filter( + (field) => message[field] !== undefined && message[field] !== null, + ).map((field) => ({ [field]: message[field] })); + const usage = json.usage as + { completion_tokens_details?: { reasoning_tokens?: number } } | undefined; + const learnedNone = + learnedReasoningDialect(this.reasoningDialectKey()) === "none"; + return { content: content ?? "", tokensUsed: json.usage?.total_tokens ?? null, @@ -328,6 +424,116 @@ export class OpenAIClient implements AIProvider { providerType: this.type, ...(toolCalls ? { toolCalls } : {}), finishReason: mapFinishReason(choice?.finish_reason), + ...(params.reasoning + ? { + reasoning: { + summary, + // Billed as completion tokens, already inside `total_tokens`. + tokens: + usage?.completion_tokens_details?.reasoning_tokens ?? null, + ...(learnedNone && params.reasoning.effort !== "off" + ? { downgradedTo: "off" as const } + : {}), + }, + } + : {}), + ...(roundTrip.length > 0 + ? { + providerState: { + providerType: this.type, + model: this.config.model, + items: roundTrip, + }, + } + : {}), }; } + + // ── v1.41 reasoning ─────────────────────────────────────────────────────── + + private reasoningDialectKey() { + return { + provider: "openai", + endpoint: this.config.baseUrl, + model: this.config.model, + }; + } + + /** + * The reasoning this request sends. Absent `params.reasoning` → none here + * (the #1126 gateway setting is applied separately, unchanged). + * + * `api.openai.com` (operator key, personal key): `reasoning_effort` only for + * a reasoning model, and only on a round without tools. Chat Completions + * has no reasoning items to hand back between tool rounds, and newer + * models restrict reasoning alongside function calls; the Coach's final + * round and the background jobs carry no tools, which is where it counts. + * + * Gateway: the endpoint's own dialect, tools or not; its reasoning state + * travels in `providerState`, and a refusal is learned per endpoint. + */ + private planReasoning( + params: CompletionParams, + hasTools: boolean, + capabilities: OpenAIChatCompletionsCapabilities, + ): OpenAIReasoningWire { + const reasoning = params.reasoning; + if (!reasoning) return { kind: "none" }; + if (learnedReasoningDialect(this.reasoningDialectKey()) === "none") { + return { kind: "none" }; + } + const level = reasoning.effort; + if (this.isGateway) { + const effort = level === "off" ? "none" : level; + return isOpenRouterEndpoint(this.config.baseUrl) + ? { kind: "openrouter", effort } + : { kind: "reasoning_effort", effort }; + } + const family = openAIReasoningFamily(this.config.model); + if ( + !family || + hasTools || + capabilities.tokenBudgetField !== "max_completion_tokens" + ) { + return { kind: "none" }; + } + return { + kind: "reasoning_effort", + effort: level === "off" ? openAIOffEffort(family) : level, + }; + } + + /** + * Put the reasoning state of earlier tool rounds back on their assistant + * messages. `buildOpenAIMessages` maps turn `i` to wire message `i + 1` + * (the system turn leads). Only state this client made for this model is + * used; anything else is dropped. + */ + private withRoundTripState( + wire: T[], + params: CompletionParams, + ): T[] { + params.messages.forEach((m, i) => { + const state = m.providerState; + if ( + m.role !== "assistant" || + !state || + state.providerType !== this.type || + state.model !== this.config.model + ) { + return; + } + const target = wire[i + 1] as Record | undefined; + if (!target) return; + for (const item of state.items) { + if (!item || typeof item !== "object") continue; + for (const field of ROUND_TRIP_FIELDS) { + if (field in item) { + target[field] = (item as Record)[field]; + } + } + } + }); + return wire; + } }