From 652dda050c62f61e5787c6e8130d941831fa9b87 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Sat, 24 Oct 2026 18:20:18 +0200 Subject: [PATCH] fix(ai): keep a model's reasoning out of the answer A gateway or local server in front of a model that always reasons (GLM behind vLLM, for one) accepts reasoning_effort "none" and ignores it. Two things went wrong with such a model. Once it answered a round with reasoning alone, the #1267 lesson asked every later call with "none", and the token ceiling followed the wire: "none" meant no thinking room, so a Coach round got its bare 600-token answer budget. The model spent that thinking and stopped at finish_reason "length" before it reached a tool call or an answer, for the hour the lesson holds. A turn set to Off had the same ceiling from the start. Where the server did not separate the reasoning (no reasoning parser, or one that falls back to content), the reply's content was the thought itself: an inline block, the text before a lone when the chat template opened the block in the prompt, or unlabelled text cut off at the ceiling. The gateway client passed all of it through as the answer; the Local client split a leading block, and only for callers that asked to reason. On these wires the ceiling now follows the call's level instead of the wire effort: Off keeps the low room, a call sent as "none" keeps its own level's room, and a call without a level gets the low room once the model has shown it reasons unasked. It is a ceiling, not a charge; a model that does switch off never touches it. Both clients now take reasoning out of content for every caller: a leading or a lone closing think tag, and a reply that filled a ceiling sized for thinking with unlabelled text and no reasoning field beside it. What comes out goes to the reasoning channel when the caller asked for it and nowhere else. A reply left without an answer takes the #1267 path and ends as reasoning_only, never as the thought. The loop test drives a Coach turn through the gateway client onto an upstream that answers like the reported wire capture, with and without a reasoning parser. Refs #1276 --- src/lib/ai/__tests__/local-reasoning.test.ts | 193 ++++++++++- ...ools-loop-always-reasoning-gateway.test.ts | 324 ++++++++++++++++++ src/lib/ai/local-client.ts | 152 +++++--- src/lib/ai/openai-client.ts | 73 +++- .../ai/reasoning/__tests__/support.test.ts | 20 ++ src/lib/ai/reasoning/levels.ts | 14 + src/lib/ai/reasoning/support.ts | 69 +++- 7 files changed, 769 insertions(+), 76 deletions(-) create mode 100644 src/lib/ai/coach/__tests__/tools-loop-always-reasoning-gateway.test.ts diff --git a/src/lib/ai/__tests__/local-reasoning.test.ts b/src/lib/ai/__tests__/local-reasoning.test.ts index aff6c797a..50565fe3c 100644 --- a/src/lib/ai/__tests__/local-reasoning.test.ts +++ b/src/lib/ai/__tests__/local-reasoning.test.ts @@ -99,7 +99,46 @@ describe("Local reasoning request", () => { expect(body(mock, 1).reasoning_effort).toBe("medium"); expect(body(mock, 1).max_tokens).toBe(600 + 4096); expect(body(mock, 2).reasoning_effort).toBe("none"); - expect(body(mock, 2).max_tokens).toBe(600); + // #1276 — `none` is a request a model may ignore: Off keeps the low room. + expect(body(mock, 2).max_tokens).toBe(600 + 1024); + }); + + it("keeps the level's room when it asks with none after a reasoning-only reply", async () => { + const mock = vi + .fn() + .mockImplementationOnce( + async () => + new Response( + JSON.stringify({ + choices: [ + { + message: { content: null, reasoning: "Thinking on and on" }, + finish_reason: "length", + }, + ], + }), + { status: 200 }, + ), + ) + .mockImplementation(buffered({ content: "ok" })); + vi.stubGlobal("fetch", mock); + const c = client(); + const wanted = params({ reasoning: { effort: "medium", summaries: true } }); + expect((await c.generateCompletion(wanted)).content).toBe("ok"); + await c.generateCompletion(wanted); + // A call without a level of its own, once the model reasons unasked. + await c.generateCompletion(params()); + expect( + [0, 1, 2, 3].map((i) => [ + body(mock, i).reasoning_effort, + body(mock, i).max_tokens, + ]), + ).toEqual([ + ["medium", 600 + 4096], + ["none", 600 + 4096], + ["none", 600 + 4096], + ["none", 600 + 1024], + ]); }); it("retries once without the field when the server refuses it, and remembers", async () => { @@ -160,7 +199,7 @@ describe("Local reasoning replies", () => { }, ); - it("splits an inline think block only for a caller that asked for reasoning", async () => { + it("splits an inline think block for every caller, and reports it only to one that asked", async () => { const reply = "check the window\n\nThe answer."; vi.stubGlobal( "fetch", @@ -171,10 +210,74 @@ describe("Local reasoning replies", () => { ); expect(asked.content).toBe("The answer."); expect(asked.reasoning?.summary).toEqual(["check the window"]); + // #1276 — reasoning is never an answer, asked for or not. const unasked = await client().generateCompletion(params()); - expect(unasked.content).toBe(reply); + expect(unasked.content).toBe("The answer."); expect(unasked.reasoning).toBeUndefined(); }); + + it("splits reasoning whose opening tag the chat template sent", async () => { + vi.stubGlobal( + "fetch", + vi.fn().mockImplementation( + buffered({ + content: "The user asks about sleep.\n\n\nDein Schlaf.", + }), + ), + ); + const result = await client().generateCompletion( + params({ reasoning: { effort: "low", summaries: true } }), + ); + expect(result.content).toBe("Dein Schlaf."); + expect(result.reasoning?.summary).toEqual(["The user asks about sleep."]); + }); + + it("never answers with unlabelled text that filled the reasoning ceiling", async () => { + const cut = async () => + new Response( + JSON.stringify({ + choices: [ + { + message: { content: "The user asks about sleep. I need to" }, + finish_reason: "length", + }, + ], + }), + { status: 200 }, + ); + const mock = vi.fn().mockImplementation(cut); + vi.stubGlobal("fetch", mock); + await expect( + client().generateCompletion( + params({ reasoning: { effort: "medium", summaries: true } }), + ), + ).rejects.toMatchObject({ httpStatus: 0, kind: "reasoning_only" }); + // Asked once more with `none` (#1267), which this model ignores. + expect(mock).toHaveBeenCalledTimes(2); + }); + + it("keeps a cut answer when the ceiling held no room for reasoning", async () => { + vi.stubGlobal( + "fetch", + vi.fn().mockImplementation( + async () => + new Response( + JSON.stringify({ + choices: [ + { + message: { content: "A long answer that ran" }, + finish_reason: "length", + }, + ], + }), + { status: 200 }, + ), + ), + ); + const result = await client().generateCompletion(params()); + expect(result.content).toBe("A long answer that ran"); + expect(result.finishReason).toBe("length"); + }); }); describe("Local reasoning stream", () => { @@ -239,7 +342,7 @@ describe("Local reasoning stream", () => { expect(result.reasoning?.summary).toEqual(["weigh the two readings"]); }); - it("streams untouched for a caller that asked for no reasoning", async () => { + it("keeps reasoning out of the answer for a caller that asked for none, and tells it nothing", async () => { vi.stubGlobal( "fetch", vi @@ -258,12 +361,90 @@ describe("Local reasoning stream", () => { params({ onReasoning }), (d) => deltas.push(d), ); - expect(deltas).toEqual(["x", "y"]); - expect(result.content).toBe("xy"); + expect(deltas).toEqual(["y"]); + expect(result.content).toBe("y"); expect(result.reasoning).toBeUndefined(); expect(onReasoning).not.toHaveBeenCalled(); }); + it("splits a closing-only think block at the end of the stream", async () => { + vi.stubGlobal( + "fetch", + vi + .fn() + .mockImplementation( + stream([ + { content: "The user asks" }, + { content: " about sleep.\n\n\nDein " }, + { content: "Schlaf." }, + ]), + ), + ); + const result = await client().generateCompletionStream( + params({ reasoning: { effort: "low", summaries: true } }), + () => {}, + ); + expect(result.content).toBe("Dein Schlaf."); + expect(result.reasoning?.summary).toEqual(["The user asks about sleep."]); + }); + + it("reads a vLLM stream: empty first content, both reasoning fields, usage on every chunk", async () => { + const encoder = new TextEncoder(); + const usage = (n: number) => ({ + prompt_tokens: 100, + completion_tokens: n, + total_tokens: 100 + n, + }); + const frames = [ + { + choices: [{ delta: { role: "assistant", content: "" } }], + usage: usage(0), + }, + { + choices: [ + { delta: { reasoning: "Plan the ", reasoning_content: "Plan the " } }, + ], + usage: usage(2), + }, + { + choices: [ + { delta: { reasoning: "answer.", reasoning_content: "answer." } }, + ], + usage: usage(3), + }, + { choices: [{ delta: { content: "Hallo." } }], usage: usage(5) }, + { choices: [{ delta: {}, finish_reason: "stop" }], usage: usage(6) }, + ]; + vi.stubGlobal( + "fetch", + vi.fn().mockImplementation( + async () => + new Response( + new ReadableStream({ + start(c) { + for (const f of frames) { + c.enqueue(encoder.encode(`data: ${JSON.stringify(f)}\n\n`)); + } + c.enqueue(encoder.encode("data: [DONE]\n\n")); + c.close(); + }, + }), + { status: 200, headers: { "content-type": "text/event-stream" } }, + ), + ), + ); + const result = await client().generateCompletionStream( + params({ reasoning: { effort: "low", summaries: true } }), + () => {}, + ); + expect(result.content).toBe("Hallo."); + // The two fields carry the same text: one copy. + expect(result.reasoning?.summary).toEqual(["Plan the answer."]); + // A usage block on every chunk is not an end: the last one counts. + expect(result.tokensUsed).toBe(106); + }); + it("passes a short reply that only looked like it might open a think block", async () => { vi.stubGlobal( "fetch", diff --git a/src/lib/ai/coach/__tests__/tools-loop-always-reasoning-gateway.test.ts b/src/lib/ai/coach/__tests__/tools-loop-always-reasoning-gateway.test.ts new file mode 100644 index 000000000..b5071cc0c --- /dev/null +++ b/src/lib/ai/coach/__tests__/tools-loop-always-reasoning-gateway.test.ts @@ -0,0 +1,324 @@ +/** + * #1276 — a Coach turn on an OpenAI-compatible gateway whose model always + * reasons and cannot be told not to (a GLM model behind vLLM). The loop runs + * through the real gateway client onto a fake upstream that answers the way + * the reported wire capture shows: + * + * - `reasoning_effort` is accepted and ignored: the model reasons on every + * request, `none` included; + * - reasoning tokens count against `max_tokens`; a budget that runs out + * inside the reasoning ends with `finish_reason: "length"`; + * - with a reasoning parser the thought arrives twice, in `reasoning` and in + * `reasoning_content`, and `content` is null beside a tool call; + * - without one, `content` carries the raw output: the reasoning, then + * `` (the chat template opened the block in the prompt), then the + * answer; a model that opens the block itself emits `` too. + * + * What must hold, however the upstream is set up: reasoning never becomes the + * answer, tool calls run, and a reply with no answer at all fails as + * `reasoning_only` instead of showing the thought. + */ +import { beforeEach, describe, expect, it, vi } from "vitest"; + +import type { CoachToolResult } from "@/lib/ai/coach/tools/run/types"; + +vi.mock("undici", async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + fetch: (input: unknown, init?: unknown) => + (globalThis.fetch as unknown as (i: unknown, n?: unknown) => unknown)( + input, + init, + ), + }; +}); +vi.mock("@/lib/ai/local-host-allowlist", () => ({ + aiEgressPolicyFor: () => ({}), +})); +const emitSignal = vi.hoisted(() => vi.fn()); +vi.mock("@/lib/logging/signal", () => ({ + emitSignal: (...a: unknown[]) => emitSignal(...a), +})); +const executeCoachTool = vi.fn<(args?: unknown) => Promise>(); +vi.mock("@/lib/ai/coach/tools/executor", () => ({ + executeCoachTool: (args: unknown) => executeCoachTool(args), +})); +const runRawCompletionWithFallback = vi.fn(); +vi.mock("@/lib/ai/provider-runner", () => ({ + runRawCompletionWithFallback: (args: unknown) => + runRawCompletionWithFallback(args), +})); + +import { runCoachToolLoop } from "@/lib/ai/coach/tools/loop"; +import { createTurnBudget } from "@/lib/ai/coach/tools/turn-budget"; +import { COACH_TOOL_DEFS } from "@/lib/ai/coach/tools/definitions"; +import { UNBOUNDED_REACH } from "@/lib/ai/coach/history-reach"; +import { AI_BUDGETS } from "@/lib/ai/ai-budgets"; +import { OpenAIClient } from "@/lib/ai/openai-client"; +import { resetReasoningDialectCache } from "@/lib/ai/reasoning/dialect-cache"; +import type { ReasoningLevel } from "@/lib/ai/reasoning/levels"; +import { singleUserTurn, type CompletionParams } from "@/lib/ai/types"; + +const BASE = "http://vllm.lan:8000/v1"; +const MODEL = "glm-5.3"; +const ARGS = JSON.stringify({ metric: "bp", window: "last30days" }); +const ANSWER = + "Dein Blutdruck lag in den letzten 30 Tagen im Mittel bei 128 zu 82."; + +/** One reasoning token per word, the way the fake counts. */ +const PLAN_WORDS = ( + "The user asks about their blood pressure. I need to fetch data with " + + "get_metric_table for the last thirty days before I can answer, then " + + "compare the readings with the earlier month and keep it short." +).split(" "); + +function thought(words: number): string { + return Array.from( + { length: words }, + (_, i) => PLAN_WORDS[i % PLAN_WORDS.length], + ).join(" "); +} + +/** How the server hands the reasoning back. */ +type Parser = + /** A reasoning parser: `reasoning` and `reasoning_content`, content null. */ + | "fielded" + /** No parser, the template opened the block: `…answer`. */ + | "template-opened" + /** No parser, the model opens the block itself: `…answer`. */ + | "raw"; + +let parser: Parser = "fielded"; +/** What the model reasons on every round, whatever it is asked. */ +let reasoningWords = 1200; +/** The finish reason the server puts beside a tool call. */ +let toolFinish: "tool_calls" | "stop" = "tool_calls"; +/** Tokens a tool call or the answer takes after the reasoning. */ +const ACT_TOKENS = 20; + +type Body = { + max_tokens?: number; + reasoning_effort?: string; + messages?: Array<{ role: string; content?: unknown }>; +}; + +function reply(message: Record, finish: string, n: number) { + return new Response( + JSON.stringify({ + id: "chatcmpl-1276", + object: "chat.completion", + model: MODEL, + choices: [{ index: 0, message, finish_reason: finish }], + usage: { + prompt_tokens: 3000, + completion_tokens: n, + total_tokens: 3000 + n, + }, + }), + { status: 200, headers: { "content-type": "application/json" } }, + ); +} + +function withReasoning( + text: string, + after: string, + extra: Record = {}, +): Record { + switch (parser) { + case "fielded": + return { + role: "assistant", + content: after === "" ? null : after, + reasoning: text, + reasoning_content: text, + ...extra, + }; + case "template-opened": + return { + role: "assistant", + content: `${text}\n\n\n${after}`, + ...extra, + }; + case "raw": + return { + role: "assistant", + content: `${text}\n\n${after}`, + ...extra, + }; + } +} + +const upstream = vi.fn(async (_url: string, init: { body: string }) => { + const body = JSON.parse(init.body) as Body; + const budget = body.max_tokens ?? 0; + if (budget < reasoningWords + ACT_TOKENS) { + // The budget runs out inside the reasoning, mid-sentence. + const partial = thought(Math.min(budget, reasoningWords)); + const message = + parser === "fielded" + ? { + role: "assistant", + content: null, + reasoning: partial, + reasoning_content: partial, + } + : parser === "raw" + ? { role: "assistant", content: `${partial}` } + : { role: "assistant", content: partial }; + return reply(message, "length", budget); + } + const used = reasoningWords + ACT_TOKENS; + const toolHistory = (body.messages ?? []).some((m) => m.role === "tool"); + if (!toolHistory) { + return reply( + withReasoning(thought(reasoningWords), "", { + tool_calls: [ + { + id: "chatcmpl-tool-1", + type: "function", + function: { name: "get_metric_table", arguments: ARGS }, + }, + ], + }), + toolFinish, + used, + ); + } + return reply(withReasoning(thought(reasoningWords), ANSWER), "stop", used); +}); + +function bodies(): Body[] { + return upstream.mock.calls.map((c) => JSON.parse(c[1].body) as Body); +} + +function gateway() { + return new OpenAIClient({ + apiKey: "", + model: MODEL, + baseUrl: BASE, + providerType: "openai-compatible", + }); +} + +/** One Coach turn as `turn/model.ts` runs it, on this gateway alone. */ +async function turn(level: ReasoningLevel) { + const client = gateway(); + runRawCompletionWithFallback.mockImplementation( + async ({ params }: { params: CompletionParams }) => ({ + result: await client.generateCompletion(params), + workingProvider: { providerType: "openai-compatible" }, + fallbackHops: [], + }), + ); + return runCoachToolLoop({ + userId: "u1", + providers: [], + system: "You are the Coach.", + messages: [{ role: "user", content: "Wie ist mein Blutdruck?" }], + tools: COACH_TOOL_DEFS, + reach: UNBOUNDED_REACH, + maxTokens: AI_BUDGETS.coach.maxTokens, + reasoning: { effort: level, summaries: level !== "off" }, + budget: createTurnBudget({ + payer: "user", + initialInputTokens: 10, + limits: { tokens: 1e9, wallMs: 1e9, maxRounds: 4 }, + }), + }); +} + +beforeEach(() => { + upstream.mockClear(); + executeCoachTool.mockReset(); + runRawCompletionWithFallback.mockReset(); + emitSignal.mockClear(); + executeCoachTool.mockResolvedValue({ present: true, data: { mean: 128 } }); + resetReasoningDialectCache(); + vi.stubGlobal("fetch", upstream); + parser = "fielded"; + reasoningWords = 1200; + toolFinish = "tool_calls"; +}); + +describe("#1276: a gateway model that always reasons", () => { + it("runs the tool and answers when its reasoning fits, with the thought kept once", async () => { + const out = await turn("medium"); + expect(out.result.content).toBe(ANSWER); + expect(executeCoachTool).toHaveBeenCalledTimes(1); + // `reasoning` and `reasoning_content` carry the same text: one copy. + expect(out.result.reasoning?.summary).toEqual([thought(1200)]); + }); + + it.each(["fielded", "template-opened", "raw"])( + "keeps room to reason after one answerless reply taught it `none` (%s)", + async (mode) => { + parser = mode; + // A long round reasons past the whole ceiling once; the #1267 rule + // learns that the model answers only unreasoned and asks again with + // `none`. This model ignores `none`, so that ask fails as well. + reasoningWords = 6000; + const first = await gateway() + .generateCompletion({ + ...singleUserTurn({ + system: "You are the Coach.", + user: "Wie ist mein Blutdruck?", + maxTokens: AI_BUDGETS.coach.maxTokens, + }), + reasoning: { effort: "medium", summaries: true }, + }) + .then( + (r) => ({ ok: true as const, content: r.content }), + (e: { kind?: string }) => ({ ok: false as const, kind: e.kind }), + ); + // Never the thought as an answer. + expect(first).toEqual({ ok: false, kind: "reasoning_only" }); + + // The next turn reasons a normal amount. + reasoningWords = 1200; + upstream.mockClear(); + const out = await turn("medium"); + expect(out.result.content).toBe(ANSWER); + expect(executeCoachTool).toHaveBeenCalledTimes(1); + for (const body of bodies()) { + expect(body.reasoning_effort).toBe("none"); + // `none` is a request, not a promise: the thinking room stays. + expect(body.max_tokens).toBe(600 + 4096); + } + }, + ); + + it("gives a turn set to Off room to reason, since the wire cannot promise off", async () => { + const out = await turn("off"); + expect(out.result.content).toBe(ANSWER); + expect(executeCoachTool).toHaveBeenCalledTimes(1); + expect(bodies()[0]).toMatchObject({ + reasoning_effort: "none", + max_tokens: 600 + 1024, + }); + }); + + it.each(["template-opened", "raw"])( + "keeps inline reasoning out of the answer and the history (%s)", + async (mode) => { + parser = mode; + const out = await turn("medium"); + expect(out.result.content).toBe(ANSWER); + expect(out.result.reasoning?.summary).toEqual([thought(1200)]); + // The assistant turn handed back on round two carries no thought. + const history = bodies()[1].messages ?? []; + const assistant = history.find((m) => m.role === "assistant"); + expect(assistant?.content ?? "").toBe(""); + }, + ); + + it("fails as reasoning_only, never with the thought, when no answer ever comes", async () => { + parser = "raw"; + reasoningWords = 20_000; + const err = await turn("medium").then( + (r) => r, + (e: unknown) => e, + ); + expect(err).toMatchObject({ httpStatus: 0, kind: "reasoning_only" }); + }); +}); diff --git a/src/lib/ai/local-client.ts b/src/lib/ai/local-client.ts index f81e4494a..6ddd46018 100644 --- a/src/lib/ai/local-client.ts +++ b/src/lib/ai/local-client.ts @@ -33,8 +33,8 @@ import { learnedReasoningDialect, rememberReasoningDialect, } from "./reasoning/dialect-cache"; -import { REASONING_THINKING_BUDGET } from "./reasoning/levels"; -import { reasoningTitleOf, splitThinkTags } from "./reasoning/support"; +import { uncontrolledThinkingRoom } from "./reasoning/levels"; +import { answerOf, reasoningTitleOf } from "./reasoning/support"; interface LocalClientConfig { apiKey?: string | null; @@ -225,6 +225,8 @@ export class LocalOpenAICompatibleClient implements AIProvider { headers: Record; body: string; jsonDialect: LocalJsonDialect; + /** The ceiling holds room for reasoning on top of the answer. */ + reasoningRoom: boolean; } { const url = `${this.config.baseUrl.replace(/\/$/, "")}/chat/completions`; // v1.28.28 (#470) — per-endpoint JSON dialect (see json-dialect.ts). @@ -251,15 +253,18 @@ export class LocalOpenAICompatibleClient implements AIProvider { ); const callEffort = this.callReasoningEffort(params); - const sentEffort = this.sentReasoningEffort(params); // Reasoning tokens count against `max_tokens` on these servers; the // thinking budget rides on top of the answer budget (a ceiling only). - const thinkingTokens = - params.reasoning && - sentEffort && - sentEffort !== "none" && - params.reasoning.effort !== "off" - ? REASONING_THINKING_BUDGET[params.reasoning.effort] + // #1276 — by the call's level, not by what the wire says: a model may + // reason whatever `reasoning_effort` asks, `none` included, and the + // `none` sent for Off or after the #1267 lesson must not leave it the + // bare answer budget to think in. A call without a level gets the `low` + // room once the model has shown it reasons unasked. + const thinkingTokens = params.reasoning + ? uncontrolledThinkingRoom(params.reasoning.effort) + : learnedReasoningDialect(this.reasoningDialectKey()) === + ANSWERS_ONLY_UNREASONED + ? uncontrolledThinkingRoom("off") : 0; const body = JSON.stringify({ model: this.config.model, @@ -298,7 +303,13 @@ export class LocalOpenAICompatibleClient implements AIProvider { : {}), }); - return { url, headers, body, jsonDialect }; + return { + url, + headers, + body, + jsonDialect, + reasoningRoom: thinkingTokens > 0, + }; } private reasoningDialectKey() { @@ -375,13 +386,52 @@ export class LocalOpenAICompatibleClient implements AIProvider { return true; } + /** + * #1276 — the answer and the reasoning of a non-streamed message, for + * every caller: `reasoning_content` / `reasoning` beside `content`, an + * inline `` block (or only its closing tag), or a reply that filled + * a ceiling sized for thinking with unlabelled text (`answerOf`). Reasoning + * never stays in `content`; only a caller that asked for it is told. + */ + private separate( + params: CompletionParams, + message: { content?: string | null } | undefined, + finishReason: string | undefined, + reasoningRoom: boolean, + ): { content: string; hadReasoning: boolean; collector: ReasoningCollector } { + const collector = new ReasoningCollector(params); + const fielded = messageReasoning(message); + if (fielded) collector.add(fielded); + const raw = message?.content ?? ""; + const separated = answerOf(raw, { + fieldedReasoning: fielded !== null, + finishReason, + hasToolCalls: false, + reasoningRoom, + }); + if (separated.reasoning) collector.add(separated.reasoning); + if (separated.unseparated) this.signalUnseparated(); + return { + content: separated.content, + hadReasoning: fielded !== null || separated.content !== raw, + collector, + }; + } + + /** A reply that was reasoning the server never separated from the answer. */ + private signalUnseparated(): void { + emitSignal({ + action: "ai.local.reasoning_unseparated", + level: "warn", + meta: { provider: "local" }, + }); + } + async generateCompletion( params: CompletionParams, ): Promise { - const { url, headers, body, jsonDialect } = this.buildRequest( - params, - false, - ); + const { url, headers, body, jsonDialect, reasoningRoom } = + this.buildRequest(params, false); // safeFetch defaults: no redirect-follow (the local endpoint is the // most exploitable on this surface — a user-controlled baseUrl that @@ -509,23 +559,14 @@ export class LocalOpenAICompatibleClient implements AIProvider { model: this.config.model, }); - const collector = new ReasoningCollector(params); - let content = json.choices?.[0]?.message?.content; - if (params.reasoning) { - const fielded = messageReasoning(json.choices?.[0]?.message); - if (fielded) collector.add(fielded); - // Only a caller that asked for reasoning gets the inline block split - // off; every other caller keeps the text exactly as it came. - if (content) { - const split = splitThinkTags(content); - if (split.reasoning) collector.add(split.reasoning); - content = split.content; - } - } const finish = json.choices?.[0]?.finish_reason; + const { content, hadReasoning, collector } = this.separate( + params, + json.choices?.[0]?.message, + finish, + reasoningRoom, + ); if (!content) { - const hadReasoning = - messageReasoning(json.choices?.[0]?.message) !== null; if (this.learnAnswerlessReasoning(params, hadReasoning, finish)) { return this.generateCompletion(params); } @@ -583,7 +624,10 @@ export class LocalOpenAICompatibleClient implements AIProvider { params: CompletionParams, onDelta: (delta: string) => void, ): Promise { - const { url, headers, body } = this.buildRequest(params, true); + const { url, headers, body, reasoningRoom } = this.buildRequest( + params, + true, + ); const egress = aiEgressPolicyFor(url, { operatorTrusted: this.config.operatorTrusted, }); @@ -646,20 +690,17 @@ export class LocalOpenAICompatibleClient implements AIProvider { label: "local AI endpoint", model: this.config.model, }); - const collector = new ReasoningCollector(params); - let buffered = json?.choices?.[0]?.message?.content; - if (params.reasoning) { - const fielded = messageReasoning(json?.choices?.[0]?.message); - if (fielded) collector.add(fielded); - if (buffered) { - const split = splitThinkTags(buffered); - if (split.reasoning) collector.add(split.reasoning); - buffered = split.content; - } - } + const { + content: buffered, + hadReasoning, + collector, + } = this.separate( + params, + json?.choices?.[0]?.message, + json?.choices?.[0]?.finish_reason, + reasoningRoom, + ); if (!buffered) { - const hadReasoning = - messageReasoning(json?.choices?.[0]?.message) !== null; if ( this.learnAnswerlessReasoning( params, @@ -697,11 +738,10 @@ export class LocalOpenAICompatibleClient implements AIProvider { // v1.41 — reasoning on the stream: the `reasoning_content` / `reasoning` // delta fields, or a leading `` block inside `content`. Inline // reasoning never reaches `onDelta`; it would otherwise show up as the - // answer. Only for a caller that asked for reasoning. + // answer. #1276 — for every caller: reasoning is never an answer, and + // only a caller that asked for it hears about it (`ReasoningCollector`). const collector = new ReasoningCollector(params); - let thinkState: "detect" | "inside" | "passthrough" = params.reasoning - ? "detect" - : "passthrough"; + let thinkState: "detect" | "inside" | "passthrough" = "detect"; let pending = ""; const passContent = (text: string) => { if (!text) return; @@ -849,6 +889,21 @@ export class LocalOpenAICompatibleClient implements AIProvider { collector.add(pending); } pending = ""; + // #1276 — what the detector cannot see while streaming: a closing tag + // with no opening one (the chat template opened the block), or a reply + // that filled a ceiling sized for thinking with unlabelled text. The + // deltas already sent are a liveness signal only (the Coach counts + // them); the returned content is what every guard reads. + const separated = answerOf(content, { + fieldedReasoning: streamedReasoning, + finishReason, + hasToolCalls: false, + reasoningRoom, + }); + if (separated.reasoning) collector.add(separated.reasoning); + if (separated.unseparated) this.signalUnseparated(); + const tookReasoning = separated.content !== content; + content = separated.content; if (!content) { // A stream that ended with a failure frame and no token reports the // failure, not "empty content". @@ -858,7 +913,8 @@ export class LocalOpenAICompatibleClient implements AIProvider { model: this.config.model, }); // An unclosed inline `` block is reasoning too. - const hadReasoning = streamedReasoning || endState === "inside"; + const hadReasoning = + streamedReasoning || endState === "inside" || tookReasoning; if (this.learnAnswerlessReasoning(params, hadReasoning, finishReason)) { return this.generateCompletionStream(params, onDelta); } diff --git a/src/lib/ai/openai-client.ts b/src/lib/ai/openai-client.ts index b4d02a07f..4f94b2035 100644 --- a/src/lib/ai/openai-client.ts +++ b/src/lib/ai/openai-client.ts @@ -15,8 +15,12 @@ import { learnedReasoningDialect, rememberReasoningDialect, } from "./reasoning/dialect-cache"; -import { REASONING_THINKING_BUDGET } from "./reasoning/levels"; import { + REASONING_THINKING_BUDGET, + uncontrolledThinkingRoom, +} from "./reasoning/levels"; +import { + answerOf, isOpenRouterEndpoint, levelOfWireEffort, openAIOffEffort, @@ -206,9 +210,22 @@ export class OpenAIClient implements AIProvider { reasoningWire.kind === "none" || reasoningWire.effort === "none" ? null : levelOfWireEffort(reasoningWire.effort); - const thinkingTokens = wireLevel - ? REASONING_THINKING_BUDGET[wireLevel === "off" ? "low" : wireLevel] - : 0; + // #1276 — on a gateway the room follows the call's level, not the wire: + // a model behind it may reason whatever `reasoning_effort` says, `none` + // included, and a `none` sent because of the #1267 lesson or the + // person's Off must not leave it the bare answer budget to think in. + // Without a level of its own, a call gets the `low` room once the model + // has shown it reasons unasked. + const thinkingTokens = this.isGateway + ? params.reasoning + ? uncontrolledThinkingRoom(params.reasoning.effort) + : learnedReasoningDialect(this.reasoningDialectKey()) === + ANSWERS_ONLY_UNREASONED + ? uncontrolledThinkingRoom("off") + : 0 + : wireLevel + ? REASONING_THINKING_BUDGET[wireLevel === "off" ? "low" : wireLevel] + : 0; const tokenBudget = (params.maxTokens ?? 1000) + thinkingTokens; const capabilityParams = capabilities.supportsSamplingControls ? { @@ -409,8 +426,37 @@ export class OpenAIClient implements AIProvider { } const choice = json.choices?.[0]; - let content = choice?.message?.content; const toolCalls = parseOpenAIToolCalls(choice); + // v1.41 — what the call reasoned. Gateways return the text as + // `reasoning` (OpenRouter, vLLM, Ollama) or `reasoning_content` + // (LiteLLM, llama.cpp), some both with the same text; `api.openai.com` + // returns none on this wire. One copy is kept. + const message = (choice?.message ?? {}) as Record; + const fieldedReasoning = [ + message.reasoning, + message.reasoning_content, + ].find((v): v is string => typeof v === "string" && v.trim().length > 0); + // #1276 — reasoning the server did not separate never becomes the + // answer: an inline `` block (or only its closing tag, when the + // chat template opened it), or a reply that filled a ceiling sized for + // thinking with unlabelled text. Only a gateway's ceiling is sized that + // way here; `api.openai.com` never inlines its reasoning. + const rawContent = choice?.message?.content ?? ""; + const separated = answerOf(rawContent, { + fieldedReasoning: fieldedReasoning !== undefined, + finishReason: choice?.finish_reason, + hasToolCalls: toolCalls !== undefined, + reasoningRoom: this.isGateway && thinkingTokens > 0, + }); + const inlineReasoning = separated.content !== rawContent; + if (separated.unseparated) { + emitSignal({ + action: "ai.gateway.reasoning_unseparated", + level: "warn", + meta: { provider: this.type }, + }); + } + let content: string = separated.content; // v1.28.28 (#470) — gateway JSON-mode shim. LiteLLM (and similar // OpenAI-compatible proxies) translate `response_format: json_object` @@ -422,7 +468,7 @@ export class OpenAIClient implements AIProvider { // into `content` so the payload parses instead of silently emptying // the insight. The real tool loop (Coach F1) always has tools and // never enters JSON mode, so it is untouched. - const trimmedContent = content?.trim() ?? ""; + const trimmedContent = content.trim(); if (useJsonFormat && (trimmedContent === "" || trimmedContent === "{}")) { const args = choice?.message?.tool_calls?.[0]?.function?.arguments; if (typeof args === "string" && args.trim().length > 0) { @@ -439,10 +485,7 @@ export class OpenAIClient implements AIProvider { // and ask again. Same rule as the Local client. const sentEffort = reasoningWire.kind !== "none" ? reasoningWire.effort : entryEffort; - const message = (choice?.message ?? {}) as Record; - const hadReasoning = [message.reasoning, message.reasoning_content].some( - (v) => typeof v === "string" && v.trim().length > 0, - ); + const hadReasoning = fieldedReasoning !== undefined || inlineReasoning; if ( this.isGateway && sentEffort !== "none" && @@ -485,13 +528,7 @@ export class OpenAIClient implements AIProvider { throw err; } - // v1.41 — what the call reasoned. Gateways return the text as - // `reasoning` (OpenRouter, vLLM, Ollama) or `reasoning_content` - // (LiteLLM, llama.cpp); `api.openai.com` returns none on this wire. - const message = (choice?.message ?? {}) as Record; - const reasoningText = [message.reasoning, message.reasoning_content].find( - (v): v is string => typeof v === "string" && v.trim().length > 0, - ); + const reasoningText = fieldedReasoning ?? separated.reasoning; const summary = reasoningText ? [reasoningText.trim()] : []; const emit = params.reasoning ? params.onReasoning : undefined; if (emit) { @@ -511,7 +548,7 @@ export class OpenAIClient implements AIProvider { learnedReasoningDialect(this.reasoningDialectKey()) === "none"; return { - content: content ?? "", + content, tokensUsed: json.usage?.total_tokens ?? null, cachedInputTokens: parseCachedTokens(json), model: this.config.model, diff --git a/src/lib/ai/reasoning/__tests__/support.test.ts b/src/lib/ai/reasoning/__tests__/support.test.ts index 97003ab72..34cfa9ef7 100644 --- a/src/lib/ai/reasoning/__tests__/support.test.ts +++ b/src/lib/ai/reasoning/__tests__/support.test.ts @@ -175,6 +175,26 @@ describe("parsing helpers", () => { }); }); + it("splits reasoning whose opening tag was part of the prompt", () => { + // GLM, Qwen3 and DeepSeek-R1 chat templates end the generation prompt + // with ``, so the reply carries only the closing tag. A server + // without a reasoning parser returns that verbatim. + expect( + splitThinkTags("The user asks about sleep.\n\n\nDein Schlaf."), + ).toEqual({ + content: "Dein Schlaf.", + reasoning: "The user asks about sleep.", + }); + expect(splitThinkTags("The answer.")).toEqual({ + content: "The answer.", + reasoning: null, + }); + expect(splitThinkTags("\nThe answer.")).toEqual({ + content: "The answer.", + reasoning: null, + }); + }); + it("maps wire efforts back onto levels", () => { expect(levelOfWireEffort("none")).toBe("off"); expect(levelOfWireEffort("minimal")).toBe("off"); diff --git a/src/lib/ai/reasoning/levels.ts b/src/lib/ai/reasoning/levels.ts index 3dca7931c..ef079470c 100644 --- a/src/lib/ai/reasoning/levels.ts +++ b/src/lib/ai/reasoning/levels.ts @@ -67,6 +67,20 @@ export const REASONING_THINKING_BUDGET: Readonly< high: 12_000, }; +/** + * #1276 — the thinking room a call gets on a wire whose reasoning switch the + * client cannot verify (an OpenAI-compatible gateway, a local server). There, + * `reasoning_effort: "none"` is a request, not a promise: a model that always + * reasons (GLM behind vLLM, say) ignores it and spends the answer budget + * thinking until `finish_reason: "length"`. So `off` keeps the `low` room, + * and a call sent as `none` because the endpoint once answered with reasoning + * alone keeps the room of its own level. A ceiling, not a charge: a model + * that does switch off never touches it. + */ +export function uncontrolledThinkingRoom(level: ReasoningLevel): number { + return REASONING_THINKING_BUDGET[level === "off" ? "low" : level]; +} + /** * The background jobs that may reason. Every other unattended surface (status * cards, batch notes, archetypes, workout and reaction lines, nudges, document diff --git a/src/lib/ai/reasoning/support.ts b/src/lib/ai/reasoning/support.ts index 20d34e642..38f41989c 100644 --- a/src/lib/ai/reasoning/support.ts +++ b/src/lib/ai/reasoning/support.ts @@ -268,21 +268,82 @@ export function reasoningTitleOf(text: string): string | null { /** * Split a reply that inlines its reasoning as a leading `…` - * block (some local models do). Only a block at the very start counts; a - * mention later in the prose stays prose. An unclosed block means the answer - * budget ran out inside the reasoning: everything is reasoning, no answer. + * block (some local models do, and any server without a reasoning parser + * passes it through). Only a block at the very start counts; a mention later + * in the prose stays prose. An unclosed block means the answer budget ran out + * inside the reasoning: everything is reasoning, no answer. + * + * #1276 — a reply can also carry only the closing tag: GLM, Qwen3 and + * DeepSeek-R1 chat templates end the generation prompt with ``, so the + * model's output starts inside the block. Everything before the first + * `` of such a reply is reasoning. */ export function splitThinkTags(content: string): { content: string; reasoning: string | null; } { - const m = /^\s*([\s\S]*?)(<\/think>|$)/.exec(content); + const m = + /^\s*([\s\S]*?)(<\/think>|$)/.exec(content) ?? + /^([\s\S]*?)(<\/think>)/.exec(content); if (!m) return { content, reasoning: null }; const reasoning = m[1].trim(); const rest = m[2] ? content.slice(m[0].length).replace(/^\s+/, "") : ""; return { content: rest, reasoning: reasoning.length > 0 ? reasoning : null }; } +/** + * #1276 — what a Chat Completions reply's `content` holds, for a wire whose + * reasoning switch the client does not control (a gateway, a local server). + * + * - Inline reasoning (`…`, or only the closing tag) is split + * off by {@link splitThinkTags}. + * - A reply that ran into a ceiling which held room for reasoning + * (`finish_reason: "length"`, `reasoningRoom`), with no reasoning field + * beside it, no tag and no tool call, is reasoning the server did not + * separate: the budget was sized for thinking plus an answer, so text + * that fills all of it unlabelled is the thought, cut off. It is never + * the answer. + * + * `reasoning` is the text taken out of `content`, for the reasoning channel. + * `unseparated` marks the second case so the caller can say so in its log. + */ +export function answerOf( + content: string | null | undefined, + reply: { + /** The reply carried `reasoning` / `reasoning_content` beside `content`. */ + fieldedReasoning: boolean; + finishReason: string | null | undefined; + hasToolCalls: boolean; + /** The request's ceiling held room for reasoning on top of the answer. */ + reasoningRoom: boolean; + }, +): { content: string; reasoning: string | null; unseparated: boolean } { + const text = content ?? ""; + if (!text) return { content: "", reasoning: null, unseparated: false }; + const split = splitThinkTags(text); + if (split.content !== text || split.reasoning !== null) { + return { + content: split.content, + reasoning: split.reasoning, + unseparated: false, + }; + } + if ( + reply.finishReason === "length" && + reply.reasoningRoom && + !reply.fieldedReasoning && + !reply.hasToolCalls + ) { + const reasoning = text.trim(); + return { + content: "", + reasoning: reasoning.length > 0 ? reasoning : null, + unseparated: true, + }; + } + return { content: text, reasoning: null, unseparated: false }; +} + /** The level a provider's wire effort corresponds to, for `downgradedTo`. */ export function levelOfWireEffort(effort: string | null): ReasoningLevel { switch (effort) {