diff --git a/src/lib/ai/__tests__/local-reasoning.test.ts b/src/lib/ai/__tests__/local-reasoning.test.ts index aff6c797a..50565fe3c 100644 --- a/src/lib/ai/__tests__/local-reasoning.test.ts +++ b/src/lib/ai/__tests__/local-reasoning.test.ts @@ -99,7 +99,46 @@ describe("Local reasoning request", () => { expect(body(mock, 1).reasoning_effort).toBe("medium"); expect(body(mock, 1).max_tokens).toBe(600 + 4096); expect(body(mock, 2).reasoning_effort).toBe("none"); - expect(body(mock, 2).max_tokens).toBe(600); + // #1276 — `none` is a request a model may ignore: Off keeps the low room. + expect(body(mock, 2).max_tokens).toBe(600 + 1024); + }); + + it("keeps the level's room when it asks with none after a reasoning-only reply", async () => { + const mock = vi + .fn() + .mockImplementationOnce( + async () => + new Response( + JSON.stringify({ + choices: [ + { + message: { content: null, reasoning: "Thinking on and on" }, + finish_reason: "length", + }, + ], + }), + { status: 200 }, + ), + ) + .mockImplementation(buffered({ content: "ok" })); + vi.stubGlobal("fetch", mock); + const c = client(); + const wanted = params({ reasoning: { effort: "medium", summaries: true } }); + expect((await c.generateCompletion(wanted)).content).toBe("ok"); + await c.generateCompletion(wanted); + // A call without a level of its own, once the model reasons unasked. + await c.generateCompletion(params()); + expect( + [0, 1, 2, 3].map((i) => [ + body(mock, i).reasoning_effort, + body(mock, i).max_tokens, + ]), + ).toEqual([ + ["medium", 600 + 4096], + ["none", 600 + 4096], + ["none", 600 + 4096], + ["none", 600 + 1024], + ]); }); it("retries once without the field when the server refuses it, and remembers", async () => { @@ -160,7 +199,7 @@ describe("Local reasoning replies", () => { }, ); - it("splits an inline think block only for a caller that asked for reasoning", async () => { + it("splits an inline think block for every caller, and reports it only to one that asked", async () => { const reply = "check the window\n\nThe answer."; vi.stubGlobal( "fetch", @@ -171,10 +210,74 @@ describe("Local reasoning replies", () => { ); expect(asked.content).toBe("The answer."); expect(asked.reasoning?.summary).toEqual(["check the window"]); + // #1276 — reasoning is never an answer, asked for or not. const unasked = await client().generateCompletion(params()); - expect(unasked.content).toBe(reply); + expect(unasked.content).toBe("The answer."); expect(unasked.reasoning).toBeUndefined(); }); + + it("splits reasoning whose opening tag the chat template sent", async () => { + vi.stubGlobal( + "fetch", + vi.fn().mockImplementation( + buffered({ + content: "The user asks about sleep.\n\n\nDein Schlaf.", + }), + ), + ); + const result = await client().generateCompletion( + params({ reasoning: { effort: "low", summaries: true } }), + ); + expect(result.content).toBe("Dein Schlaf."); + expect(result.reasoning?.summary).toEqual(["The user asks about sleep."]); + }); + + it("never answers with unlabelled text that filled the reasoning ceiling", async () => { + const cut = async () => + new Response( + JSON.stringify({ + choices: [ + { + message: { content: "The user asks about sleep. I need to" }, + finish_reason: "length", + }, + ], + }), + { status: 200 }, + ); + const mock = vi.fn().mockImplementation(cut); + vi.stubGlobal("fetch", mock); + await expect( + client().generateCompletion( + params({ reasoning: { effort: "medium", summaries: true } }), + ), + ).rejects.toMatchObject({ httpStatus: 0, kind: "reasoning_only" }); + // Asked once more with `none` (#1267), which this model ignores. + expect(mock).toHaveBeenCalledTimes(2); + }); + + it("keeps a cut answer when the ceiling held no room for reasoning", async () => { + vi.stubGlobal( + "fetch", + vi.fn().mockImplementation( + async () => + new Response( + JSON.stringify({ + choices: [ + { + message: { content: "A long answer that ran" }, + finish_reason: "length", + }, + ], + }), + { status: 200 }, + ), + ), + ); + const result = await client().generateCompletion(params()); + expect(result.content).toBe("A long answer that ran"); + expect(result.finishReason).toBe("length"); + }); }); describe("Local reasoning stream", () => { @@ -239,7 +342,7 @@ describe("Local reasoning stream", () => { expect(result.reasoning?.summary).toEqual(["weigh the two readings"]); }); - it("streams untouched for a caller that asked for no reasoning", async () => { + it("keeps reasoning out of the answer for a caller that asked for none, and tells it nothing", async () => { vi.stubGlobal( "fetch", vi @@ -258,12 +361,90 @@ describe("Local reasoning stream", () => { params({ onReasoning }), (d) => deltas.push(d), ); - expect(deltas).toEqual(["x", "y"]); - expect(result.content).toBe("xy"); + expect(deltas).toEqual(["y"]); + expect(result.content).toBe("y"); expect(result.reasoning).toBeUndefined(); expect(onReasoning).not.toHaveBeenCalled(); }); + it("splits a closing-only think block at the end of the stream", async () => { + vi.stubGlobal( + "fetch", + vi + .fn() + .mockImplementation( + stream([ + { content: "The user asks" }, + { content: " about sleep.\n\n\nDein " }, + { content: "Schlaf." }, + ]), + ), + ); + const result = await client().generateCompletionStream( + params({ reasoning: { effort: "low", summaries: true } }), + () => {}, + ); + expect(result.content).toBe("Dein Schlaf."); + expect(result.reasoning?.summary).toEqual(["The user asks about sleep."]); + }); + + it("reads a vLLM stream: empty first content, both reasoning fields, usage on every chunk", async () => { + const encoder = new TextEncoder(); + const usage = (n: number) => ({ + prompt_tokens: 100, + completion_tokens: n, + total_tokens: 100 + n, + }); + const frames = [ + { + choices: [{ delta: { role: "assistant", content: "" } }], + usage: usage(0), + }, + { + choices: [ + { delta: { reasoning: "Plan the ", reasoning_content: "Plan the " } }, + ], + usage: usage(2), + }, + { + choices: [ + { delta: { reasoning: "answer.", reasoning_content: "answer." } }, + ], + usage: usage(3), + }, + { choices: [{ delta: { content: "Hallo." } }], usage: usage(5) }, + { choices: [{ delta: {}, finish_reason: "stop" }], usage: usage(6) }, + ]; + vi.stubGlobal( + "fetch", + vi.fn().mockImplementation( + async () => + new Response( + new ReadableStream({ + start(c) { + for (const f of frames) { + c.enqueue(encoder.encode(`data: ${JSON.stringify(f)}\n\n`)); + } + c.enqueue(encoder.encode("data: [DONE]\n\n")); + c.close(); + }, + }), + { status: 200, headers: { "content-type": "text/event-stream" } }, + ), + ), + ); + const result = await client().generateCompletionStream( + params({ reasoning: { effort: "low", summaries: true } }), + () => {}, + ); + expect(result.content).toBe("Hallo."); + // The two fields carry the same text: one copy. + expect(result.reasoning?.summary).toEqual(["Plan the answer."]); + // A usage block on every chunk is not an end: the last one counts. + expect(result.tokensUsed).toBe(106); + }); + it("passes a short reply that only looked like it might open a think block", async () => { vi.stubGlobal( "fetch", diff --git a/src/lib/ai/coach/__tests__/tools-loop-always-reasoning-gateway.test.ts b/src/lib/ai/coach/__tests__/tools-loop-always-reasoning-gateway.test.ts new file mode 100644 index 000000000..b5071cc0c --- /dev/null +++ b/src/lib/ai/coach/__tests__/tools-loop-always-reasoning-gateway.test.ts @@ -0,0 +1,324 @@ +/** + * #1276 — a Coach turn on an OpenAI-compatible gateway whose model always + * reasons and cannot be told not to (a GLM model behind vLLM). The loop runs + * through the real gateway client onto a fake upstream that answers the way + * the reported wire capture shows: + * + * - `reasoning_effort` is accepted and ignored: the model reasons on every + * request, `none` included; + * - reasoning tokens count against `max_tokens`; a budget that runs out + * inside the reasoning ends with `finish_reason: "length"`; + * - with a reasoning parser the thought arrives twice, in `reasoning` and in + * `reasoning_content`, and `content` is null beside a tool call; + * - without one, `content` carries the raw output: the reasoning, then + * `` (the chat template opened the block in the prompt), then the + * answer; a model that opens the block itself emits `` too. + * + * What must hold, however the upstream is set up: reasoning never becomes the + * answer, tool calls run, and a reply with no answer at all fails as + * `reasoning_only` instead of showing the thought. + */ +import { beforeEach, describe, expect, it, vi } from "vitest"; + +import type { CoachToolResult } from "@/lib/ai/coach/tools/run/types"; + +vi.mock("undici", async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + fetch: (input: unknown, init?: unknown) => + (globalThis.fetch as unknown as (i: unknown, n?: unknown) => unknown)( + input, + init, + ), + }; +}); +vi.mock("@/lib/ai/local-host-allowlist", () => ({ + aiEgressPolicyFor: () => ({}), +})); +const emitSignal = vi.hoisted(() => vi.fn()); +vi.mock("@/lib/logging/signal", () => ({ + emitSignal: (...a: unknown[]) => emitSignal(...a), +})); +const executeCoachTool = vi.fn<(args?: unknown) => Promise>(); +vi.mock("@/lib/ai/coach/tools/executor", () => ({ + executeCoachTool: (args: unknown) => executeCoachTool(args), +})); +const runRawCompletionWithFallback = vi.fn(); +vi.mock("@/lib/ai/provider-runner", () => ({ + runRawCompletionWithFallback: (args: unknown) => + runRawCompletionWithFallback(args), +})); + +import { runCoachToolLoop } from "@/lib/ai/coach/tools/loop"; +import { createTurnBudget } from "@/lib/ai/coach/tools/turn-budget"; +import { COACH_TOOL_DEFS } from "@/lib/ai/coach/tools/definitions"; +import { UNBOUNDED_REACH } from "@/lib/ai/coach/history-reach"; +import { AI_BUDGETS } from "@/lib/ai/ai-budgets"; +import { OpenAIClient } from "@/lib/ai/openai-client"; +import { resetReasoningDialectCache } from "@/lib/ai/reasoning/dialect-cache"; +import type { ReasoningLevel } from "@/lib/ai/reasoning/levels"; +import { singleUserTurn, type CompletionParams } from "@/lib/ai/types"; + +const BASE = "http://vllm.lan:8000/v1"; +const MODEL = "glm-5.3"; +const ARGS = JSON.stringify({ metric: "bp", window: "last30days" }); +const ANSWER = + "Dein Blutdruck lag in den letzten 30 Tagen im Mittel bei 128 zu 82."; + +/** One reasoning token per word, the way the fake counts. */ +const PLAN_WORDS = ( + "The user asks about their blood pressure. I need to fetch data with " + + "get_metric_table for the last thirty days before I can answer, then " + + "compare the readings with the earlier month and keep it short." +).split(" "); + +function thought(words: number): string { + return Array.from( + { length: words }, + (_, i) => PLAN_WORDS[i % PLAN_WORDS.length], + ).join(" "); +} + +/** How the server hands the reasoning back. */ +type Parser = + /** A reasoning parser: `reasoning` and `reasoning_content`, content null. */ + | "fielded" + /** No parser, the template opened the block: `…answer`. */ + | "template-opened" + /** No parser, the model opens the block itself: `…answer`. */ + | "raw"; + +let parser: Parser = "fielded"; +/** What the model reasons on every round, whatever it is asked. */ +let reasoningWords = 1200; +/** The finish reason the server puts beside a tool call. */ +let toolFinish: "tool_calls" | "stop" = "tool_calls"; +/** Tokens a tool call or the answer takes after the reasoning. */ +const ACT_TOKENS = 20; + +type Body = { + max_tokens?: number; + reasoning_effort?: string; + messages?: Array<{ role: string; content?: unknown }>; +}; + +function reply(message: Record, finish: string, n: number) { + return new Response( + JSON.stringify({ + id: "chatcmpl-1276", + object: "chat.completion", + model: MODEL, + choices: [{ index: 0, message, finish_reason: finish }], + usage: { + prompt_tokens: 3000, + completion_tokens: n, + total_tokens: 3000 + n, + }, + }), + { status: 200, headers: { "content-type": "application/json" } }, + ); +} + +function withReasoning( + text: string, + after: string, + extra: Record = {}, +): Record { + switch (parser) { + case "fielded": + return { + role: "assistant", + content: after === "" ? null : after, + reasoning: text, + reasoning_content: text, + ...extra, + }; + case "template-opened": + return { + role: "assistant", + content: `${text}\n\n\n${after}`, + ...extra, + }; + case "raw": + return { + role: "assistant", + content: `${text}\n\n${after}`, + ...extra, + }; + } +} + +const upstream = vi.fn(async (_url: string, init: { body: string }) => { + const body = JSON.parse(init.body) as Body; + const budget = body.max_tokens ?? 0; + if (budget < reasoningWords + ACT_TOKENS) { + // The budget runs out inside the reasoning, mid-sentence. + const partial = thought(Math.min(budget, reasoningWords)); + const message = + parser === "fielded" + ? { + role: "assistant", + content: null, + reasoning: partial, + reasoning_content: partial, + } + : parser === "raw" + ? { role: "assistant", content: `${partial}` } + : { role: "assistant", content: partial }; + return reply(message, "length", budget); + } + const used = reasoningWords + ACT_TOKENS; + const toolHistory = (body.messages ?? []).some((m) => m.role === "tool"); + if (!toolHistory) { + return reply( + withReasoning(thought(reasoningWords), "", { + tool_calls: [ + { + id: "chatcmpl-tool-1", + type: "function", + function: { name: "get_metric_table", arguments: ARGS }, + }, + ], + }), + toolFinish, + used, + ); + } + return reply(withReasoning(thought(reasoningWords), ANSWER), "stop", used); +}); + +function bodies(): Body[] { + return upstream.mock.calls.map((c) => JSON.parse(c[1].body) as Body); +} + +function gateway() { + return new OpenAIClient({ + apiKey: "", + model: MODEL, + baseUrl: BASE, + providerType: "openai-compatible", + }); +} + +/** One Coach turn as `turn/model.ts` runs it, on this gateway alone. */ +async function turn(level: ReasoningLevel) { + const client = gateway(); + runRawCompletionWithFallback.mockImplementation( + async ({ params }: { params: CompletionParams }) => ({ + result: await client.generateCompletion(params), + workingProvider: { providerType: "openai-compatible" }, + fallbackHops: [], + }), + ); + return runCoachToolLoop({ + userId: "u1", + providers: [], + system: "You are the Coach.", + messages: [{ role: "user", content: "Wie ist mein Blutdruck?" }], + tools: COACH_TOOL_DEFS, + reach: UNBOUNDED_REACH, + maxTokens: AI_BUDGETS.coach.maxTokens, + reasoning: { effort: level, summaries: level !== "off" }, + budget: createTurnBudget({ + payer: "user", + initialInputTokens: 10, + limits: { tokens: 1e9, wallMs: 1e9, maxRounds: 4 }, + }), + }); +} + +beforeEach(() => { + upstream.mockClear(); + executeCoachTool.mockReset(); + runRawCompletionWithFallback.mockReset(); + emitSignal.mockClear(); + executeCoachTool.mockResolvedValue({ present: true, data: { mean: 128 } }); + resetReasoningDialectCache(); + vi.stubGlobal("fetch", upstream); + parser = "fielded"; + reasoningWords = 1200; + toolFinish = "tool_calls"; +}); + +describe("#1276: a gateway model that always reasons", () => { + it("runs the tool and answers when its reasoning fits, with the thought kept once", async () => { + const out = await turn("medium"); + expect(out.result.content).toBe(ANSWER); + expect(executeCoachTool).toHaveBeenCalledTimes(1); + // `reasoning` and `reasoning_content` carry the same text: one copy. + expect(out.result.reasoning?.summary).toEqual([thought(1200)]); + }); + + it.each(["fielded", "template-opened", "raw"])( + "keeps room to reason after one answerless reply taught it `none` (%s)", + async (mode) => { + parser = mode; + // A long round reasons past the whole ceiling once; the #1267 rule + // learns that the model answers only unreasoned and asks again with + // `none`. This model ignores `none`, so that ask fails as well. + reasoningWords = 6000; + const first = await gateway() + .generateCompletion({ + ...singleUserTurn({ + system: "You are the Coach.", + user: "Wie ist mein Blutdruck?", + maxTokens: AI_BUDGETS.coach.maxTokens, + }), + reasoning: { effort: "medium", summaries: true }, + }) + .then( + (r) => ({ ok: true as const, content: r.content }), + (e: { kind?: string }) => ({ ok: false as const, kind: e.kind }), + ); + // Never the thought as an answer. + expect(first).toEqual({ ok: false, kind: "reasoning_only" }); + + // The next turn reasons a normal amount. + reasoningWords = 1200; + upstream.mockClear(); + const out = await turn("medium"); + expect(out.result.content).toBe(ANSWER); + expect(executeCoachTool).toHaveBeenCalledTimes(1); + for (const body of bodies()) { + expect(body.reasoning_effort).toBe("none"); + // `none` is a request, not a promise: the thinking room stays. + expect(body.max_tokens).toBe(600 + 4096); + } + }, + ); + + it("gives a turn set to Off room to reason, since the wire cannot promise off", async () => { + const out = await turn("off"); + expect(out.result.content).toBe(ANSWER); + expect(executeCoachTool).toHaveBeenCalledTimes(1); + expect(bodies()[0]).toMatchObject({ + reasoning_effort: "none", + max_tokens: 600 + 1024, + }); + }); + + it.each(["template-opened", "raw"])( + "keeps inline reasoning out of the answer and the history (%s)", + async (mode) => { + parser = mode; + const out = await turn("medium"); + expect(out.result.content).toBe(ANSWER); + expect(out.result.reasoning?.summary).toEqual([thought(1200)]); + // The assistant turn handed back on round two carries no thought. + const history = bodies()[1].messages ?? []; + const assistant = history.find((m) => m.role === "assistant"); + expect(assistant?.content ?? "").toBe(""); + }, + ); + + it("fails as reasoning_only, never with the thought, when no answer ever comes", async () => { + parser = "raw"; + reasoningWords = 20_000; + const err = await turn("medium").then( + (r) => r, + (e: unknown) => e, + ); + expect(err).toMatchObject({ httpStatus: 0, kind: "reasoning_only" }); + }); +}); diff --git a/src/lib/ai/local-client.ts b/src/lib/ai/local-client.ts index f81e4494a..6ddd46018 100644 --- a/src/lib/ai/local-client.ts +++ b/src/lib/ai/local-client.ts @@ -33,8 +33,8 @@ import { learnedReasoningDialect, rememberReasoningDialect, } from "./reasoning/dialect-cache"; -import { REASONING_THINKING_BUDGET } from "./reasoning/levels"; -import { reasoningTitleOf, splitThinkTags } from "./reasoning/support"; +import { uncontrolledThinkingRoom } from "./reasoning/levels"; +import { answerOf, reasoningTitleOf } from "./reasoning/support"; interface LocalClientConfig { apiKey?: string | null; @@ -225,6 +225,8 @@ export class LocalOpenAICompatibleClient implements AIProvider { headers: Record; body: string; jsonDialect: LocalJsonDialect; + /** The ceiling holds room for reasoning on top of the answer. */ + reasoningRoom: boolean; } { const url = `${this.config.baseUrl.replace(/\/$/, "")}/chat/completions`; // v1.28.28 (#470) — per-endpoint JSON dialect (see json-dialect.ts). @@ -251,15 +253,18 @@ export class LocalOpenAICompatibleClient implements AIProvider { ); const callEffort = this.callReasoningEffort(params); - const sentEffort = this.sentReasoningEffort(params); // Reasoning tokens count against `max_tokens` on these servers; the // thinking budget rides on top of the answer budget (a ceiling only). - const thinkingTokens = - params.reasoning && - sentEffort && - sentEffort !== "none" && - params.reasoning.effort !== "off" - ? REASONING_THINKING_BUDGET[params.reasoning.effort] + // #1276 — by the call's level, not by what the wire says: a model may + // reason whatever `reasoning_effort` asks, `none` included, and the + // `none` sent for Off or after the #1267 lesson must not leave it the + // bare answer budget to think in. A call without a level gets the `low` + // room once the model has shown it reasons unasked. + const thinkingTokens = params.reasoning + ? uncontrolledThinkingRoom(params.reasoning.effort) + : learnedReasoningDialect(this.reasoningDialectKey()) === + ANSWERS_ONLY_UNREASONED + ? uncontrolledThinkingRoom("off") : 0; const body = JSON.stringify({ model: this.config.model, @@ -298,7 +303,13 @@ export class LocalOpenAICompatibleClient implements AIProvider { : {}), }); - return { url, headers, body, jsonDialect }; + return { + url, + headers, + body, + jsonDialect, + reasoningRoom: thinkingTokens > 0, + }; } private reasoningDialectKey() { @@ -375,13 +386,52 @@ export class LocalOpenAICompatibleClient implements AIProvider { return true; } + /** + * #1276 — the answer and the reasoning of a non-streamed message, for + * every caller: `reasoning_content` / `reasoning` beside `content`, an + * inline `` block (or only its closing tag), or a reply that filled + * a ceiling sized for thinking with unlabelled text (`answerOf`). Reasoning + * never stays in `content`; only a caller that asked for it is told. + */ + private separate( + params: CompletionParams, + message: { content?: string | null } | undefined, + finishReason: string | undefined, + reasoningRoom: boolean, + ): { content: string; hadReasoning: boolean; collector: ReasoningCollector } { + const collector = new ReasoningCollector(params); + const fielded = messageReasoning(message); + if (fielded) collector.add(fielded); + const raw = message?.content ?? ""; + const separated = answerOf(raw, { + fieldedReasoning: fielded !== null, + finishReason, + hasToolCalls: false, + reasoningRoom, + }); + if (separated.reasoning) collector.add(separated.reasoning); + if (separated.unseparated) this.signalUnseparated(); + return { + content: separated.content, + hadReasoning: fielded !== null || separated.content !== raw, + collector, + }; + } + + /** A reply that was reasoning the server never separated from the answer. */ + private signalUnseparated(): void { + emitSignal({ + action: "ai.local.reasoning_unseparated", + level: "warn", + meta: { provider: "local" }, + }); + } + async generateCompletion( params: CompletionParams, ): Promise { - const { url, headers, body, jsonDialect } = this.buildRequest( - params, - false, - ); + const { url, headers, body, jsonDialect, reasoningRoom } = + this.buildRequest(params, false); // safeFetch defaults: no redirect-follow (the local endpoint is the // most exploitable on this surface — a user-controlled baseUrl that @@ -509,23 +559,14 @@ export class LocalOpenAICompatibleClient implements AIProvider { model: this.config.model, }); - const collector = new ReasoningCollector(params); - let content = json.choices?.[0]?.message?.content; - if (params.reasoning) { - const fielded = messageReasoning(json.choices?.[0]?.message); - if (fielded) collector.add(fielded); - // Only a caller that asked for reasoning gets the inline block split - // off; every other caller keeps the text exactly as it came. - if (content) { - const split = splitThinkTags(content); - if (split.reasoning) collector.add(split.reasoning); - content = split.content; - } - } const finish = json.choices?.[0]?.finish_reason; + const { content, hadReasoning, collector } = this.separate( + params, + json.choices?.[0]?.message, + finish, + reasoningRoom, + ); if (!content) { - const hadReasoning = - messageReasoning(json.choices?.[0]?.message) !== null; if (this.learnAnswerlessReasoning(params, hadReasoning, finish)) { return this.generateCompletion(params); } @@ -583,7 +624,10 @@ export class LocalOpenAICompatibleClient implements AIProvider { params: CompletionParams, onDelta: (delta: string) => void, ): Promise { - const { url, headers, body } = this.buildRequest(params, true); + const { url, headers, body, reasoningRoom } = this.buildRequest( + params, + true, + ); const egress = aiEgressPolicyFor(url, { operatorTrusted: this.config.operatorTrusted, }); @@ -646,20 +690,17 @@ export class LocalOpenAICompatibleClient implements AIProvider { label: "local AI endpoint", model: this.config.model, }); - const collector = new ReasoningCollector(params); - let buffered = json?.choices?.[0]?.message?.content; - if (params.reasoning) { - const fielded = messageReasoning(json?.choices?.[0]?.message); - if (fielded) collector.add(fielded); - if (buffered) { - const split = splitThinkTags(buffered); - if (split.reasoning) collector.add(split.reasoning); - buffered = split.content; - } - } + const { + content: buffered, + hadReasoning, + collector, + } = this.separate( + params, + json?.choices?.[0]?.message, + json?.choices?.[0]?.finish_reason, + reasoningRoom, + ); if (!buffered) { - const hadReasoning = - messageReasoning(json?.choices?.[0]?.message) !== null; if ( this.learnAnswerlessReasoning( params, @@ -697,11 +738,10 @@ export class LocalOpenAICompatibleClient implements AIProvider { // v1.41 — reasoning on the stream: the `reasoning_content` / `reasoning` // delta fields, or a leading `` block inside `content`. Inline // reasoning never reaches `onDelta`; it would otherwise show up as the - // answer. Only for a caller that asked for reasoning. + // answer. #1276 — for every caller: reasoning is never an answer, and + // only a caller that asked for it hears about it (`ReasoningCollector`). const collector = new ReasoningCollector(params); - let thinkState: "detect" | "inside" | "passthrough" = params.reasoning - ? "detect" - : "passthrough"; + let thinkState: "detect" | "inside" | "passthrough" = "detect"; let pending = ""; const passContent = (text: string) => { if (!text) return; @@ -849,6 +889,21 @@ export class LocalOpenAICompatibleClient implements AIProvider { collector.add(pending); } pending = ""; + // #1276 — what the detector cannot see while streaming: a closing tag + // with no opening one (the chat template opened the block), or a reply + // that filled a ceiling sized for thinking with unlabelled text. The + // deltas already sent are a liveness signal only (the Coach counts + // them); the returned content is what every guard reads. + const separated = answerOf(content, { + fieldedReasoning: streamedReasoning, + finishReason, + hasToolCalls: false, + reasoningRoom, + }); + if (separated.reasoning) collector.add(separated.reasoning); + if (separated.unseparated) this.signalUnseparated(); + const tookReasoning = separated.content !== content; + content = separated.content; if (!content) { // A stream that ended with a failure frame and no token reports the // failure, not "empty content". @@ -858,7 +913,8 @@ export class LocalOpenAICompatibleClient implements AIProvider { model: this.config.model, }); // An unclosed inline `` block is reasoning too. - const hadReasoning = streamedReasoning || endState === "inside"; + const hadReasoning = + streamedReasoning || endState === "inside" || tookReasoning; if (this.learnAnswerlessReasoning(params, hadReasoning, finishReason)) { return this.generateCompletionStream(params, onDelta); } diff --git a/src/lib/ai/openai-client.ts b/src/lib/ai/openai-client.ts index b4d02a07f..4f94b2035 100644 --- a/src/lib/ai/openai-client.ts +++ b/src/lib/ai/openai-client.ts @@ -15,8 +15,12 @@ import { learnedReasoningDialect, rememberReasoningDialect, } from "./reasoning/dialect-cache"; -import { REASONING_THINKING_BUDGET } from "./reasoning/levels"; import { + REASONING_THINKING_BUDGET, + uncontrolledThinkingRoom, +} from "./reasoning/levels"; +import { + answerOf, isOpenRouterEndpoint, levelOfWireEffort, openAIOffEffort, @@ -206,9 +210,22 @@ export class OpenAIClient implements AIProvider { reasoningWire.kind === "none" || reasoningWire.effort === "none" ? null : levelOfWireEffort(reasoningWire.effort); - const thinkingTokens = wireLevel - ? REASONING_THINKING_BUDGET[wireLevel === "off" ? "low" : wireLevel] - : 0; + // #1276 — on a gateway the room follows the call's level, not the wire: + // a model behind it may reason whatever `reasoning_effort` says, `none` + // included, and a `none` sent because of the #1267 lesson or the + // person's Off must not leave it the bare answer budget to think in. + // Without a level of its own, a call gets the `low` room once the model + // has shown it reasons unasked. + const thinkingTokens = this.isGateway + ? params.reasoning + ? uncontrolledThinkingRoom(params.reasoning.effort) + : learnedReasoningDialect(this.reasoningDialectKey()) === + ANSWERS_ONLY_UNREASONED + ? uncontrolledThinkingRoom("off") + : 0 + : wireLevel + ? REASONING_THINKING_BUDGET[wireLevel === "off" ? "low" : wireLevel] + : 0; const tokenBudget = (params.maxTokens ?? 1000) + thinkingTokens; const capabilityParams = capabilities.supportsSamplingControls ? { @@ -409,8 +426,37 @@ export class OpenAIClient implements AIProvider { } const choice = json.choices?.[0]; - let content = choice?.message?.content; const toolCalls = parseOpenAIToolCalls(choice); + // v1.41 — what the call reasoned. Gateways return the text as + // `reasoning` (OpenRouter, vLLM, Ollama) or `reasoning_content` + // (LiteLLM, llama.cpp), some both with the same text; `api.openai.com` + // returns none on this wire. One copy is kept. + const message = (choice?.message ?? {}) as Record; + const fieldedReasoning = [ + message.reasoning, + message.reasoning_content, + ].find((v): v is string => typeof v === "string" && v.trim().length > 0); + // #1276 — reasoning the server did not separate never becomes the + // answer: an inline `` block (or only its closing tag, when the + // chat template opened it), or a reply that filled a ceiling sized for + // thinking with unlabelled text. Only a gateway's ceiling is sized that + // way here; `api.openai.com` never inlines its reasoning. + const rawContent = choice?.message?.content ?? ""; + const separated = answerOf(rawContent, { + fieldedReasoning: fieldedReasoning !== undefined, + finishReason: choice?.finish_reason, + hasToolCalls: toolCalls !== undefined, + reasoningRoom: this.isGateway && thinkingTokens > 0, + }); + const inlineReasoning = separated.content !== rawContent; + if (separated.unseparated) { + emitSignal({ + action: "ai.gateway.reasoning_unseparated", + level: "warn", + meta: { provider: this.type }, + }); + } + let content: string = separated.content; // v1.28.28 (#470) — gateway JSON-mode shim. LiteLLM (and similar // OpenAI-compatible proxies) translate `response_format: json_object` @@ -422,7 +468,7 @@ export class OpenAIClient implements AIProvider { // into `content` so the payload parses instead of silently emptying // the insight. The real tool loop (Coach F1) always has tools and // never enters JSON mode, so it is untouched. - const trimmedContent = content?.trim() ?? ""; + const trimmedContent = content.trim(); if (useJsonFormat && (trimmedContent === "" || trimmedContent === "{}")) { const args = choice?.message?.tool_calls?.[0]?.function?.arguments; if (typeof args === "string" && args.trim().length > 0) { @@ -439,10 +485,7 @@ export class OpenAIClient implements AIProvider { // and ask again. Same rule as the Local client. const sentEffort = reasoningWire.kind !== "none" ? reasoningWire.effort : entryEffort; - const message = (choice?.message ?? {}) as Record; - const hadReasoning = [message.reasoning, message.reasoning_content].some( - (v) => typeof v === "string" && v.trim().length > 0, - ); + const hadReasoning = fieldedReasoning !== undefined || inlineReasoning; if ( this.isGateway && sentEffort !== "none" && @@ -485,13 +528,7 @@ export class OpenAIClient implements AIProvider { throw err; } - // v1.41 — what the call reasoned. Gateways return the text as - // `reasoning` (OpenRouter, vLLM, Ollama) or `reasoning_content` - // (LiteLLM, llama.cpp); `api.openai.com` returns none on this wire. - const message = (choice?.message ?? {}) as Record; - const reasoningText = [message.reasoning, message.reasoning_content].find( - (v): v is string => typeof v === "string" && v.trim().length > 0, - ); + const reasoningText = fieldedReasoning ?? separated.reasoning; const summary = reasoningText ? [reasoningText.trim()] : []; const emit = params.reasoning ? params.onReasoning : undefined; if (emit) { @@ -511,7 +548,7 @@ export class OpenAIClient implements AIProvider { learnedReasoningDialect(this.reasoningDialectKey()) === "none"; return { - content: content ?? "", + content, tokensUsed: json.usage?.total_tokens ?? null, cachedInputTokens: parseCachedTokens(json), model: this.config.model, diff --git a/src/lib/ai/reasoning/__tests__/support.test.ts b/src/lib/ai/reasoning/__tests__/support.test.ts index 97003ab72..34cfa9ef7 100644 --- a/src/lib/ai/reasoning/__tests__/support.test.ts +++ b/src/lib/ai/reasoning/__tests__/support.test.ts @@ -175,6 +175,26 @@ describe("parsing helpers", () => { }); }); + it("splits reasoning whose opening tag was part of the prompt", () => { + // GLM, Qwen3 and DeepSeek-R1 chat templates end the generation prompt + // with ``, so the reply carries only the closing tag. A server + // without a reasoning parser returns that verbatim. + expect( + splitThinkTags("The user asks about sleep.\n\n\nDein Schlaf."), + ).toEqual({ + content: "Dein Schlaf.", + reasoning: "The user asks about sleep.", + }); + expect(splitThinkTags("The answer.")).toEqual({ + content: "The answer.", + reasoning: null, + }); + expect(splitThinkTags("\nThe answer.")).toEqual({ + content: "The answer.", + reasoning: null, + }); + }); + it("maps wire efforts back onto levels", () => { expect(levelOfWireEffort("none")).toBe("off"); expect(levelOfWireEffort("minimal")).toBe("off"); diff --git a/src/lib/ai/reasoning/levels.ts b/src/lib/ai/reasoning/levels.ts index 3dca7931c..ef079470c 100644 --- a/src/lib/ai/reasoning/levels.ts +++ b/src/lib/ai/reasoning/levels.ts @@ -67,6 +67,20 @@ export const REASONING_THINKING_BUDGET: Readonly< high: 12_000, }; +/** + * #1276 — the thinking room a call gets on a wire whose reasoning switch the + * client cannot verify (an OpenAI-compatible gateway, a local server). There, + * `reasoning_effort: "none"` is a request, not a promise: a model that always + * reasons (GLM behind vLLM, say) ignores it and spends the answer budget + * thinking until `finish_reason: "length"`. So `off` keeps the `low` room, + * and a call sent as `none` because the endpoint once answered with reasoning + * alone keeps the room of its own level. A ceiling, not a charge: a model + * that does switch off never touches it. + */ +export function uncontrolledThinkingRoom(level: ReasoningLevel): number { + return REASONING_THINKING_BUDGET[level === "off" ? "low" : level]; +} + /** * The background jobs that may reason. Every other unattended surface (status * cards, batch notes, archetypes, workout and reaction lines, nudges, document diff --git a/src/lib/ai/reasoning/support.ts b/src/lib/ai/reasoning/support.ts index 20d34e642..38f41989c 100644 --- a/src/lib/ai/reasoning/support.ts +++ b/src/lib/ai/reasoning/support.ts @@ -268,21 +268,82 @@ export function reasoningTitleOf(text: string): string | null { /** * Split a reply that inlines its reasoning as a leading `…` - * block (some local models do). Only a block at the very start counts; a - * mention later in the prose stays prose. An unclosed block means the answer - * budget ran out inside the reasoning: everything is reasoning, no answer. + * block (some local models do, and any server without a reasoning parser + * passes it through). Only a block at the very start counts; a mention later + * in the prose stays prose. An unclosed block means the answer budget ran out + * inside the reasoning: everything is reasoning, no answer. + * + * #1276 — a reply can also carry only the closing tag: GLM, Qwen3 and + * DeepSeek-R1 chat templates end the generation prompt with ``, so the + * model's output starts inside the block. Everything before the first + * `` of such a reply is reasoning. */ export function splitThinkTags(content: string): { content: string; reasoning: string | null; } { - const m = /^\s*([\s\S]*?)(<\/think>|$)/.exec(content); + const m = + /^\s*([\s\S]*?)(<\/think>|$)/.exec(content) ?? + /^([\s\S]*?)(<\/think>)/.exec(content); if (!m) return { content, reasoning: null }; const reasoning = m[1].trim(); const rest = m[2] ? content.slice(m[0].length).replace(/^\s+/, "") : ""; return { content: rest, reasoning: reasoning.length > 0 ? reasoning : null }; } +/** + * #1276 — what a Chat Completions reply's `content` holds, for a wire whose + * reasoning switch the client does not control (a gateway, a local server). + * + * - Inline reasoning (`…`, or only the closing tag) is split + * off by {@link splitThinkTags}. + * - A reply that ran into a ceiling which held room for reasoning + * (`finish_reason: "length"`, `reasoningRoom`), with no reasoning field + * beside it, no tag and no tool call, is reasoning the server did not + * separate: the budget was sized for thinking plus an answer, so text + * that fills all of it unlabelled is the thought, cut off. It is never + * the answer. + * + * `reasoning` is the text taken out of `content`, for the reasoning channel. + * `unseparated` marks the second case so the caller can say so in its log. + */ +export function answerOf( + content: string | null | undefined, + reply: { + /** The reply carried `reasoning` / `reasoning_content` beside `content`. */ + fieldedReasoning: boolean; + finishReason: string | null | undefined; + hasToolCalls: boolean; + /** The request's ceiling held room for reasoning on top of the answer. */ + reasoningRoom: boolean; + }, +): { content: string; reasoning: string | null; unseparated: boolean } { + const text = content ?? ""; + if (!text) return { content: "", reasoning: null, unseparated: false }; + const split = splitThinkTags(text); + if (split.content !== text || split.reasoning !== null) { + return { + content: split.content, + reasoning: split.reasoning, + unseparated: false, + }; + } + if ( + reply.finishReason === "length" && + reply.reasoningRoom && + !reply.fieldedReasoning && + !reply.hasToolCalls + ) { + const reasoning = text.trim(); + return { + content: "", + reasoning: reasoning.length > 0 ? reasoning : null, + unseparated: true, + }; + } + return { content: text, reasoning: null, unseparated: false }; +} + /** The level a provider's wire effort corresponds to, for `downgradedTo`. */ export function levelOfWireEffort(effort: string | null): ReasoningLevel { switch (effort) {