diff --git a/src/__tests__/baselines/source-file-lines.json b/src/__tests__/baselines/source-file-lines.json index 8b53d058b..d57271904 100644 --- a/src/__tests__/baselines/source-file-lines.json +++ b/src/__tests__/baselines/source-file-lines.json @@ -1,7 +1,7 @@ { "$comment": "Production files over 800 lines and their length. Maintained by src/__tests__/source-size-ratchet.test.ts.", - "count": 87, - "total": 97452, + "count": 86, + "total": 96631, "entries": { "app/medications/page-client.tsx": 839, "components/charts/mood-chart.tsx": 1166, @@ -29,7 +29,6 @@ "lib/ai/coach/stream-events.ts": 817, "lib/ai/coach/types.ts": 1011, "lib/ai/codex-client.ts": 1045, - "lib/ai/local-client.ts": 821, "lib/ai/prompts/insight-generator.ts": 1110, "lib/ai/provider.ts": 1632, "lib/analytics/score/reader.ts": 998, diff --git a/src/__tests__/baselines/source-function-lines.json b/src/__tests__/baselines/source-function-lines.json index e91851790..7e0885638 100644 --- a/src/__tests__/baselines/source-function-lines.json +++ b/src/__tests__/baselines/source-function-lines.json @@ -1,7 +1,7 @@ { "$comment": "Production functions over 200 lines, keyed \"file :: name\", and their length. Maintained by src/__tests__/source-size-ratchet.test.ts.", "count": 351, - "total": 113581, + "total": 113530, "entries": { "app/achievements/page.tsx :: AchievementsPage": 206, "app/api/admin/backups/upload/route.ts :: apiHandler(arg)": 314, @@ -254,8 +254,8 @@ "lib/ai/coach/turn/pipeline.ts :: runTurn": 274, "lib/ai/coach/turn/reply-guards.ts :: guardReply": 331, "lib/ai/codex-client.ts :: parseResponse": 302, - "lib/ai/local-client.ts :: generateCompletionStream": 263, - "lib/ai/openai-client.ts :: generateCompletion": 276, + "lib/ai/local-client.ts :: generateCompletionStream": 222, + "lib/ai/openai-client.ts :: generateCompletion": 266, "lib/analytics/bp-in-target-fast-path.ts :: computeFromRollups": 203, "lib/analytics/compliance/display.ts :: calculateCompliance": 212, "lib/analytics/correlations-fast-path.ts :: computeCorrelationHypothesesFastPath": 256, diff --git a/src/__tests__/baselines/version-stamp-comments.json b/src/__tests__/baselines/version-stamp-comments.json index 512d97513..064d48b73 100644 --- a/src/__tests__/baselines/version-stamp-comments.json +++ b/src/__tests__/baselines/version-stamp-comments.json @@ -1,7 +1,7 @@ { "$comment": "Comment lines that open with a release version, per file under src/. Maintained by src/__tests__/version-stamp-ratchet.test.ts.", "count": 2897, - "total": 9267, + "total": 9263, "entries": { "__tests__/account-payload-consumer-guard.test.ts": 2, "__tests__/acting-account-boundary-guard.test.ts": 7, @@ -1751,7 +1751,7 @@ "lib/ai/local-client.ts": 16, "lib/ai/medical-references.ts": 1, "lib/ai/mock-client.ts": 5, - "lib/ai/openai-client.ts": 13, + "lib/ai/openai-client.ts": 9, "lib/ai/prompts/__tests__/assessment-prompt-contract.test.ts": 5, "lib/ai/prompts/__tests__/briefing-personalisation.test.ts": 1, "lib/ai/prompts/__tests__/coach-prompt-locale.test.ts": 1, diff --git a/src/lib/ai/__tests__/local-reasoning.test.ts b/src/lib/ai/__tests__/local-reasoning.test.ts index 50565fe3c..30e5f091f 100644 --- a/src/lib/ai/__tests__/local-reasoning.test.ts +++ b/src/lib/ai/__tests__/local-reasoning.test.ts @@ -232,7 +232,7 @@ describe("Local reasoning replies", () => { expect(result.reasoning?.summary).toEqual(["The user asks about sleep."]); }); - it("never answers with unlabelled text that filled the reasoning ceiling", async () => { + it("never answers with unlabelled text that filled the reasoning ceiling, once the model reasons", async () => { const cut = async () => new Response( JSON.stringify({ @@ -245,15 +245,47 @@ describe("Local reasoning replies", () => { }), { status: 200 }, ); - const mock = vi.fn().mockImplementation(cut); + // The model has answered with a reasoning field before. + const mock = vi + .fn() + .mockImplementationOnce( + buffered({ content: "ok", reasoning: "Thinking first" }), + ) + .mockImplementation(cut); vi.stubGlobal("fetch", mock); - await expect( - client().generateCompletion( - params({ reasoning: { effort: "medium", summaries: true } }), - ), - ).rejects.toMatchObject({ httpStatus: 0, kind: "reasoning_only" }); + const c = client(); + const wanted = params({ reasoning: { effort: "medium", summaries: true } }); + expect((await c.generateCompletion(wanted)).content).toBe("ok"); + await expect(c.generateCompletion(wanted)).rejects.toMatchObject({ + httpStatus: 0, + kind: "reasoning_only", + }); // Asked once more with `none` (#1267), which this model ignores. - expect(mock).toHaveBeenCalledTimes(2); + expect(mock).toHaveBeenCalledTimes(3); + }); + + it("keeps a cut answer from a model never seen reasoning, whatever the level", async () => { + const mock = vi.fn().mockImplementation( + async () => + new Response( + JSON.stringify({ + choices: [ + { + message: { content: '{"summary": "Steady and' }, + finish_reason: "length", + }, + ], + }), + { status: 200 }, + ), + ); + vi.stubGlobal("fetch", mock); + const result = await client().generateCompletion( + params({ reasoning: { effort: "medium", summaries: true } }), + ); + expect(result.content).toBe('{"summary": "Steady and'); + expect(result.finishReason).toBe("length"); + expect(mock).toHaveBeenCalledTimes(1); }); it("keeps a cut answer when the ceiling held no room for reasoning", async () => { diff --git a/src/lib/ai/__tests__/truncated-plain-answer.test.ts b/src/lib/ai/__tests__/truncated-plain-answer.test.ts new file mode 100644 index 000000000..4ab9ea976 --- /dev/null +++ b/src/lib/ai/__tests__/truncated-plain-answer.test.ts @@ -0,0 +1,178 @@ +/** + * #1276 — a reply that ran into the token ceiling with unlabelled text is the + * model's answer, cut off, unless this endpoint and model have shown that + * they reason. + * + * The rule read every such reply as reasoning the server did not separate, + * whenever the ceiling held room for reasoning: a call carrying any level + * (Off included) to a model that does not reason at all (llama3.1) lost its + * own truncated answer. The briefing never reached its JSON-shape retry, the + * Coach named the wrong error, the failure counted toward the bench, and the + * client learned to ask with `none` for an hour. Now the text is reasoning + * only once the model has shown it reasons (a reasoning field, a think tag, + * or a lesson that says so); otherwise it is the answer with + * `finishReason: "length"`. + * + * Driven through the real clients against a loopback server. + */ +import http from "node:http"; +import type { AddressInfo } from "node:net"; +import { + afterAll, + afterEach, + beforeAll, + beforeEach, + describe, + expect, + it, + vi, +} from "vitest"; + +vi.mock("@/lib/logging/signal", () => ({ emitSignal: () => {} })); + +import { LocalOpenAICompatibleClient } from "../local-client"; +import { OpenAIClient } from "../openai-client"; +import { resetJsonModeDialectCache } from "../json-dialect"; +import { + learnedReasoningDialect, + resetReasoningDialectCache, +} from "../reasoning/dialect-cache"; +import { singleUserTurn, type CompletionParams } from "../types"; +import { _resetPinnedDispatcherForTests } from "@/lib/safe-fetch-dispatcher"; + +const CUT = '{"summary": "Your weight has been stable over the last weeks and'; + +/** What the next replies carry, one per request; the last one repeats. */ +let replies: Array> = []; +let seen: unknown[] = []; +let server: http.Server; +let origin = ""; + +function cut() { + return { message: { role: "assistant", content: CUT }, finish: "length" }; +} + +function reasonedAnswer() { + return { + message: { + role: "assistant", + content: '{"summary":"Steady."}', + reasoning_content: "The readings are flat.", + }, + finish: "stop", + }; +} + +beforeAll(async () => { + server = http.createServer((req, res) => { + let raw = ""; + req.on("data", (c) => (raw += c)); + req.on("end", () => { + const body = JSON.parse(raw) as Record; + seen.push(body.reasoning_effort); + const next = (replies.length > 1 ? replies.shift() : replies[0]) as { + message: unknown; + finish: string; + }; + res.writeHead(200, { "Content-Type": "application/json" }); + res.end( + JSON.stringify({ + choices: [ + { index: 0, message: next.message, finish_reason: next.finish }, + ], + usage: { total_tokens: 3000 }, + }), + ); + }); + }); + await new Promise((r) => server.listen(0, "127.0.0.1", r)); + origin = `http://127.0.0.1:${(server.address() as AddressInfo).port}`; +}); + +afterAll(async () => { + _resetPinnedDispatcherForTests(); + await new Promise((r) => server.close(() => r())); +}); + +const savedOrigins = process.env.AI_PRIVATE_ORIGINS; + +beforeEach(() => { + process.env.AI_PRIVATE_ORIGINS = origin; + seen = []; + replies = []; + resetReasoningDialectCache(); + resetJsonModeDialectCache(); +}); + +afterEach(() => { + if (savedOrigins === undefined) delete process.env.AI_PRIVATE_ORIGINS; + else process.env.AI_PRIVATE_ORIGINS = savedOrigins; +}); + +function params(effort: "off" | "low"): CompletionParams { + return { + ...singleUserTurn({ + system: "s", + user: "u", + maxTokens: 2500, + timeoutMs: 30_000, + responseFormat: "json", + }), + reasoning: { effort, summaries: false }, + }; +} + +describe.each([ + [ + "the gateway", + () => + new OpenAIClient({ + apiKey: "", + model: "llama3.1", + baseUrl: `${origin}/v1`, + providerType: "openai-compatible", + }), + "openai", + ], + [ + "the Local client", + () => + new LocalOpenAICompatibleClient({ + baseUrl: `${origin}/v1`, + model: "llama3.1", + }), + "local", + ], +] as const)("%s", (_, make, provider) => { + const key = () => ({ + provider, + endpoint: `${origin}/v1`, + model: "llama3.1", + }); + + it.each(["off", "low"] as const)( + "returns a model's cut answer as the answer (level %s), and learns nothing", + async (effort) => { + replies = [cut()]; + for (let i = 0; i < 2; i += 1) { + const result = await make().generateCompletion(params(effort)); + expect(result.content).toBe(CUT); + expect(result.finishReason).toBe("length"); + } + // One request per call, each sent as the call asked. + const asked = effort === "off" ? "none" : "low"; + expect(seen).toEqual([asked, asked]); + expect(learnedReasoningDialect(key())).toBeUndefined(); + }, + ); + + it("reads the same cut text as reasoning once the model has shown it reasons", async () => { + replies = [reasonedAnswer(), cut()]; + const first = await make().generateCompletion(params("low")); + expect(first.content).toBe('{"summary":"Steady."}'); + const err = await make() + .generateCompletion(params("low")) + .catch((e: unknown) => e as Record); + expect(err).toMatchObject({ httpStatus: 0, kind: "reasoning_only" }); + }); +}); diff --git a/src/lib/ai/coach/__tests__/tools-loop-always-reasoning-gateway.test.ts b/src/lib/ai/coach/__tests__/tools-loop-always-reasoning-gateway.test.ts index e66828695..29c02bae1 100644 --- a/src/lib/ai/coach/__tests__/tools-loop-always-reasoning-gateway.test.ts +++ b/src/lib/ai/coach/__tests__/tools-loop-always-reasoning-gateway.test.ts @@ -268,11 +268,20 @@ describe("#1276: a gateway model that always reasons", () => { reasoning: { effort: "medium", summaries: true }, }) .then( - (r) => ({ ok: true as const, content: r.content }), + (r) => ({ ok: true as const, finishReason: r.finishReason }), (e: { kind?: string }) => ({ ok: false as const, kind: e.kind }), ); - // Never the thought as an answer. - expect(first).toEqual({ ok: false, kind: "reasoning_only" }); + // Never the thought as an answer, once it is labelled as one. The + // template-opened reply carries no label at all (no field, no tag) and + // comes from a model not yet seen reasoning: text cut off at the + // ceiling from such a model is its answer, cut off (a model that does + // not reason, asked with any level, must keep its answer), so it comes + // back as the answer with `finishReason: "length"`. + expect(first).toEqual( + mode === "template-opened" + ? { ok: true, finishReason: "length" } + : { ok: false, kind: "reasoning_only" }, + ); // The next turn reasons a normal amount. reasoningWords = 1200; diff --git a/src/lib/ai/local-client.ts b/src/lib/ai/local-client.ts index 52fa08b11..239602e3c 100644 --- a/src/lib/ai/local-client.ts +++ b/src/lib/ai/local-client.ts @@ -25,18 +25,22 @@ import type { ReasoningEffort } from "./reasoning-effort"; import { ANSWERS_ONLY_UNREASONED, learnedReasoningDialect, + noteReasoningSeen, refusesReasoningField, } from "./reasoning/dialect-cache"; import { uncontrolledThinkingRoom } from "./reasoning/levels"; import { askAgainUnreasoned, + knownReasoner, reasonsUnasked, repeatUnreasoned, } from "./reasoning/answerless"; import { ReasoningCollector, + StreamedThinkSplitter, separateMessage, takeReasoning, + type LocalReasoner, } from "./local-reasoning"; interface LocalClientConfig { @@ -230,6 +234,22 @@ export class LocalOpenAICompatibleClient implements AIProvider { return params.reasoning.effort === "off" ? "none" : params.reasoning.effort; } + /** + * #1276 — what this endpoint and model have shown of their reasoning, for + * one request (`LocalReasoner`). Seen on a request that asked to reason + * counts only for requests that ask too. + */ + private reasoner(params: CompletionParams, room: boolean): LocalReasoner { + const key = this.reasoningDialectKey(); + const sent = this.sentReasoningEffort(params); + const asked = sent !== null && sent !== "none"; + return { + room, + known: knownReasoner(key, asked), + seen: () => noteReasoningSeen(key, asked), + }; + } + /** * The `reasoning_effort` this request carries, whichever level decided it: * the call's own (`callReasoningEffort`), else the entry's #1126 setting. @@ -436,7 +456,7 @@ export class LocalOpenAICompatibleClient implements AIProvider { params, json.choices?.[0]?.message, finish, - room, + this.reasoner(params, room), ); if (!content) { const spent = json.usage?.total_tokens ?? null; @@ -506,7 +526,7 @@ export class LocalOpenAICompatibleClient implements AIProvider { params, json?.choices?.[0]?.message, json?.choices?.[0]?.finish_reason, - room, + this.reasoner(params, room), ); if (!content) { const spent = json?.usage?.total_tokens ?? null; @@ -626,44 +646,12 @@ export class LocalOpenAICompatibleClient implements AIProvider { // reasoning never reaches `onDelta`; it would otherwise show up as the // answer. #1276 — for every caller (only one that asked is told). const collector = new ReasoningCollector(params); - let thinkState: "detect" | "inside" | "passthrough" = "detect"; - let pending = ""; - const passContent = (text: string) => { + const splitter = new StreamedThinkSplitter(collector, (text) => { if (!text) return; collector.finish(); content += text; onDelta(text); - }; - const onContent = (delta: string) => { - if (thinkState === "passthrough") { - passContent(delta); - return; - } - pending += delta; - if (thinkState === "detect") { - const lead = pending.replace(/^\s+/, ""); - if (lead.length < "".length && "".startsWith(lead)) { - return; - } - if (!lead.startsWith("")) { - thinkState = "passthrough"; - const out = pending; - pending = ""; - passContent(out); - return; - } - thinkState = "inside"; - pending = lead.slice("".length); - } - const close = pending.indexOf(""); - if (close === -1) return; - collector.add(pending.slice(0, close)); - collector.finish(); - const rest = pending.slice(close + "".length).replace(/^\s+/, ""); - pending = ""; - thinkState = "passthrough"; - passContent(rest); - }; + }); // An SSE frame can carry the same embedded failure the buffered body // carries. Remember the first one; it only matters if the stream ends @@ -731,7 +719,7 @@ export class LocalOpenAICompatibleClient implements AIProvider { collector.add(reasoningDelta); } const delta = choiceDelta?.content; - if (delta) onContent(delta); + if (delta) splitter.push(delta); if (typeof chunk.usage?.total_tokens === "number") { tokensUsed = chunk.usage.total_tokens; } @@ -763,24 +751,16 @@ export class LocalOpenAICompatibleClient implements AIProvider { reader.cancel().catch(() => {}); } - // A reply short enough to still sit in the `` detector is prose; - // an unclosed block is reasoning that ran out of budget. - // (The closures above move `thinkState`; read it afresh.) - const endState = thinkState as "detect" | "inside" | "passthrough"; - if (endState === "detect") { - thinkState = "passthrough"; - passContent(pending); - } else if (endState === "inside") { - collector.add(pending); - } - pending = ""; + const { unclosed } = splitter.end(); // #1276 — a lone ``, or unlabelled text that filled the ceiling. // Sent deltas are a liveness signal only; the guards read `content`. + const reasoner = this.reasoner(params, room); const tail = takeReasoning(collector, content, { fieldedReasoning: streamedReasoning, finishReason, - reasoningRoom: room, + reasoner, }); + if (streamedReasoning || splitter.opened || tail.took) reasoner.seen(); content = tail.content; if (!content) { // A stream that ended with a failure frame and no token reports the @@ -791,8 +771,7 @@ export class LocalOpenAICompatibleClient implements AIProvider { model: this.config.model, }); // An unclosed inline `` block is reasoning too. - const hadReasoning = - streamedReasoning || endState === "inside" || tail.took; + const hadReasoning = streamedReasoning || unclosed || tail.took; if ( this.learnAnswerlessReasoning(params, hops, hadReasoning, finishReason) ) { diff --git a/src/lib/ai/local-reasoning.ts b/src/lib/ai/local-reasoning.ts index 7584bfb38..124686018 100644 --- a/src/lib/ai/local-reasoning.ts +++ b/src/lib/ai/local-reasoning.ts @@ -74,12 +74,23 @@ export function messageReasoning(message: unknown): string | null { return text ?? null; } +/** + * What one request's endpoint and model have shown of their reasoning + * (#1276): `room` the ceiling held room for it, `known` the model was seen + * reasoning before (`knownReasoner`), `seen` records a reply that reasoned. + */ +export interface LocalReasoner { + room: boolean; + known: boolean; + seen(): void; +} + /** * #1276 — take the reasoning out of `content` (`answerOf`: an inline - * `` block or only its closing tag, or unlabelled text that filled a - * ceiling sized for thinking) and hand it to `collector`. Returns the answer - * and whether anything was taken. For every caller: reasoning is never an - * answer, asked for or not. + * `` block or only its closing tag, or, from a model known to reason, + * unlabelled text that filled a ceiling sized for thinking) and hand it to + * `collector`. Returns the answer and whether anything was taken. For every + * caller: reasoning is never an answer, asked for or not. */ export function takeReasoning( collector: ReasoningCollector, @@ -87,10 +98,16 @@ export function takeReasoning( reply: { fieldedReasoning: boolean; finishReason: string | undefined; - reasoningRoom: boolean; + reasoner: LocalReasoner; }, ): { content: string; took: boolean } { - const separated = answerOf(content, { ...reply, hasToolCalls: false }); + const separated = answerOf(content, { + fieldedReasoning: reply.fieldedReasoning, + finishReason: reply.finishReason, + hasToolCalls: false, + reasoningRoom: reply.reasoner.room, + knownReasoner: reply.reasoner.known, + }); if (separated.reasoning) collector.add(separated.reasoning); if (separated.unseparated) { emitSignal({ @@ -110,7 +127,7 @@ export function separateMessage( params: CompletionParams, message: { content?: string | null } | undefined, finishReason: string | undefined, - reasoningRoom: boolean, + reasoner: LocalReasoner, ): { content: string; hadReasoning: boolean; collector: ReasoningCollector } { const collector = new ReasoningCollector(params); const fielded = messageReasoning(message); @@ -118,7 +135,77 @@ export function separateMessage( const { content, took } = takeReasoning(collector, message?.content ?? "", { fieldedReasoning: fielded !== null, finishReason, - reasoningRoom, + reasoner, }); - return { content, hadReasoning: fielded !== null || took, collector }; + const hadReasoning = fielded !== null || took; + if (hadReasoning) reasoner.seen(); + return { content, hadReasoning, collector }; +} + +/** + * A leading inline `` block in streamed `content`, split off delta by + * delta: the reasoning goes to the collector, never to `pass`, which gets the + * answer as it arrives. + */ +export class StreamedThinkSplitter { + private state: "detect" | "inside" | "passthrough" = "detect"; + private pending = ""; + /** A `` block opened. */ + opened = false; + + constructor( + private readonly collector: ReasoningCollector, + private readonly pass: (text: string) => void, + ) {} + + push(delta: string): void { + if (this.state === "passthrough") { + this.pass(delta); + return; + } + this.pending += delta; + if (this.state === "detect") { + const lead = this.pending.replace(/^\s+/, ""); + if (lead.length < "".length && "".startsWith(lead)) { + return; + } + if (!lead.startsWith("")) { + this.state = "passthrough"; + const out = this.pending; + this.pending = ""; + this.pass(out); + return; + } + this.state = "inside"; + this.opened = true; + this.pending = lead.slice("".length); + } + const close = this.pending.indexOf(""); + if (close === -1) return; + this.collector.add(this.pending.slice(0, close)); + this.collector.finish(); + const rest = this.pending + .slice(close + "".length) + .replace(/^\s+/, ""); + this.pending = ""; + this.state = "passthrough"; + this.pass(rest); + } + + /** + * The stream ended. A reply short enough to still sit in the detector is + * prose; an unclosed block is reasoning that ran out of budget, and the + * result says so. + */ + end(): { unclosed: boolean } { + const unclosed = this.state === "inside"; + if (this.state === "detect") { + this.state = "passthrough"; + this.pass(this.pending); + } else if (unclosed) { + this.collector.add(this.pending); + } + this.pending = ""; + return { unclosed }; + } } diff --git a/src/lib/ai/openai-client.ts b/src/lib/ai/openai-client.ts index f2bd442e2..a05a4a913 100644 --- a/src/lib/ai/openai-client.ts +++ b/src/lib/ai/openai-client.ts @@ -15,10 +15,12 @@ import { import { ANSWERS_ONLY_UNREASONED, learnedReasoningDialect, + noteReasoningSeen, refusesReasoningField, } from "./reasoning/dialect-cache"; import { askAgainUnreasoned, + knownReasoner, reasonsUnasked, repeatUnreasoned, } from "./reasoning/answerless"; @@ -276,30 +278,15 @@ export class OpenAIClient implements AIProvider { : {}), }), }, - // 60 s ceiling so a tar-pit upstream cannot pin a worker - // indefinitely. Real completions land well inside this budget. - // v1.11.2 — the base URL is user/admin-overridable (BYO gateway), so pin - // the connect-time DNS check: a base URL resolving to a private/metadata - // address is rejected, closing the SSRF/rebinding surface. - // v1.20.1 — compose the caller's cancel signal (Coach SSE disconnect) so - // a mid-generation abort tears the upstream call down early. - // The record owner's response-timeout setting, else the surface value, else 60 s. - // v1.33.1 (#470) — the gateway tag reuses the Local provider's host - // policy verbatim: public hosts always; a private host only when the - // operator granted its origin (`AI_PRIVATE_ORIGINS` or the legacy host - // list), and then only through the pinned operator-approved dispatcher. - // - // v1.37.30 — the `admin-key` tag consults the same policy. Its base URL - // is the one other operator-typed URL in this client (an operator can - // point the global provider at a private OpenAI-compatible proxy, e.g. - // an OAuth sidecar), and both ends of the grant are the operator's own: - // the URL comes from the admin settings, the allowlist from the - // operator's environment. Every non-operator construction of this tag - // pins the public `api.openai.com` constant, for which the policy - // resolves to `true` without ever consulting the allowlist — so the - // personal-key posture is unchanged. The `codex` tag keeps the hard - // pin: its base URL is a repository constant, there is nothing an - // operator legitimately redirects. + // The record owner's response-timeout setting (else the surface value, + // else 60 s), and the caller's cancel signal (a Coach disconnect tears + // the upstream call down). The gateway's base URL and the operator's + // `admin-key` one are typed by a person, so they take the Local + // provider's host policy: public hosts always, a private host only for + // an origin the operator granted (`AI_PRIVATE_ORIGINS`), dialled through + // the pinned operator-approved dispatcher; a personal key pins the + // public `api.openai.com`, for which the policy never consults the + // allowlist. `codex` keeps the hard pin: its URL is a repository constant. { timeoutMs: callTimeoutMs(params, this.responseTimeoutSeconds), ...(this.isGateway || this.type === "admin-key" @@ -357,7 +344,13 @@ export class OpenAIClient implements AIProvider { const choice = json.choices?.[0]; const toolCalls = parseOpenAIToolCalls(choice); - const reply = this.separateReasoning(choice, toolCalls, thinkingTokens); + // The effort the request carried, whichever level decided it. + const sent = + reasoningWire.kind !== "none" ? reasoningWire.effort : entryEffort; + const reply = this.separateReasoning(choice, toolCalls, { + thinkingTokens, + sent, + }); let content: string = reply.content; // v1.28.28 (#470) — gateway JSON-mode shim. LiteLLM (and similar @@ -382,8 +375,7 @@ export class OpenAIClient implements AIProvider { // empty reply with neither content NOR tool calls is an error. if (!content && !toolCalls) { return this.answerless(params, hops, { - sent: - reasoningWire.kind !== "none" ? reasoningWire.effort : entryEffort, + sent, hadReasoning: reply.hadReasoning, finishReason: choice?.finish_reason, spent: json.usage?.total_tokens ?? null, @@ -613,19 +605,24 @@ export class OpenAIClient implements AIProvider { private separateReasoning( choice: OpenAIChoice | undefined, toolCalls: AiToolCall[] | undefined, - thinkingTokens: number, + request: { thinkingTokens: number; sent: string | null }, ): { content: string; reasoning: string | null; hadReasoning: boolean } { const message = (choice?.message ?? {}) as Record; const fielded = [message.reasoning, message.reasoning_content].find( (v): v is string => typeof v === "string" && v.trim().length > 0, ); const raw = choice?.message?.content ?? ""; + const key = this.reasoningDialectKey(); + const asked = request.sent !== null && request.sent !== "none"; const separated = answerOf(raw, { fieldedReasoning: fielded !== undefined, finishReason: choice?.finish_reason, hasToolCalls: toolCalls !== undefined, - reasoningRoom: this.isGateway && thinkingTokens > 0, + reasoningRoom: this.isGateway && request.thinkingTokens > 0, + knownReasoner: knownReasoner(key, asked), }); + const hadReasoning = fielded !== undefined || separated.content !== raw; + if (hadReasoning) noteReasoningSeen(key, asked); if (separated.unseparated) { emitSignal({ action: "ai.gateway.reasoning_unseparated", @@ -636,7 +633,7 @@ export class OpenAIClient implements AIProvider { return { content: separated.content, reasoning: fielded ?? separated.reasoning, - hadReasoning: fielded !== undefined || separated.content !== raw, + hadReasoning, }; } diff --git a/src/lib/ai/reasoning/__tests__/support.test.ts b/src/lib/ai/reasoning/__tests__/support.test.ts index 34cfa9ef7..73999c462 100644 --- a/src/lib/ai/reasoning/__tests__/support.test.ts +++ b/src/lib/ai/reasoning/__tests__/support.test.ts @@ -1,5 +1,6 @@ import { describe, expect, it } from "vitest"; import { + answerOf, anthropicModelVersion, anthropicRefusesSampling, anthropicThinkingDialect, @@ -195,6 +196,56 @@ describe("parsing helpers", () => { }); }); + it("leaves a closing tag mentioned in a sentence alone, unless the model is known to reason", () => { + const prose = + "Some models end their thinking with before they answer."; + expect(splitThinkTags(prose)).toEqual({ content: prose, reasoning: null }); + // The same text from a model seen reasoning: the template opened the + // block, so the tag closes it wherever it sits. + expect(splitThinkTags("Weighing the weekSteady.", true)).toEqual({ + content: "Steady.", + reasoning: "Weighing the week", + }); + // A tag that opens a line splits whoever sent it. + expect(splitThinkTags("Weighing the week\n \nSteady.")).toEqual({ + content: "Steady.", + reasoning: "Weighing the week", + }); + }); + + describe("answerOf, a reply cut off at the ceiling", () => { + const cut = '{"summary": "Your weight has been stable and'; + const reply = (knownReasoner: boolean) => ({ + fieldedReasoning: false, + finishReason: "length", + hasToolCalls: false, + reasoningRoom: true, + knownReasoner, + }); + + it("is the answer from a model not known to reason", () => { + expect(answerOf(cut, reply(false))).toEqual({ + content: cut, + reasoning: null, + unseparated: false, + }); + }); + + it("is reasoning the server did not separate from a model known to reason", () => { + expect(answerOf(cut, reply(true))).toEqual({ + content: "", + reasoning: cut, + unseparated: true, + }); + }); + + it("is the answer when the ceiling held no room for reasoning", () => { + expect( + answerOf(cut, { ...reply(true), reasoningRoom: false }).content, + ).toBe(cut); + }); + }); + it("maps wire efforts back onto levels", () => { expect(levelOfWireEffort("none")).toBe("off"); expect(levelOfWireEffort("minimal")).toBe("off"); diff --git a/src/lib/ai/reasoning/answerless.ts b/src/lib/ai/reasoning/answerless.ts index 57e8c835f..fdbb86e48 100644 --- a/src/lib/ai/reasoning/answerless.ts +++ b/src/lib/ai/reasoning/answerless.ts @@ -31,6 +31,7 @@ import { REASONS_REGARDLESS, annotateReasoningDowngrade, learnedReasoningDialect, + reasoningSeen, rememberReasoningDialect, type ReasoningDialectKey, } from "./dialect-cache"; @@ -135,6 +136,19 @@ export function reasonsUnasked(key: ReasoningDialectKey): boolean { ); } +/** + * #1276 — true when the model has shown it reasons, for a request that did + * (`asked`) or did not ask it to: a lesson that says it reasons unasked, or a + * reply seen reasoning (`reasoningSeen`). Only then is unlabelled text that + * ran into the ceiling read as reasoning (`answerOf`). + */ +export function knownReasoner( + key: ReasoningDialectKey, + asked: boolean, +): boolean { + return reasonsUnasked(key) || reasoningSeen(key, asked); +} + /** * The endpoint refused the reasoning field `sent`: send none from now on. A * model already known to reason unasked keeps that lesson too, so it is never diff --git a/src/lib/ai/reasoning/dialect-cache.ts b/src/lib/ai/reasoning/dialect-cache.ts index c190ae940..b1d7f841c 100644 --- a/src/lib/ai/reasoning/dialect-cache.ts +++ b/src/lib/ai/reasoning/dialect-cache.ts @@ -150,9 +150,58 @@ export function isReasoningParameterRejection( return params.test(text) && PARAMETER_REFUSAL.test(text); } +/** + * #1276 — endpoints and models seen reasoning: a reasoning field or a think + * tag in a reply. `unasked` when the request carried no effort (or `none`), + * and it outlasts a later sighting on a request that asked. Held as long as a + * lesson (`DIALECT_TTL_MS`) and on `globalThis` for the same reason. + */ +const SEEN_SLOT = Symbol.for("healthlog.ai.reasoningSeen"); +const seen: Map = (( + globalThis as Record +)[SEEN_SLOT] ??= new Map()) as Map< + string, + { unasked: boolean; expiresAt: number } +>; + +function seenEntry(k: string) { + const entry = seen.get(k); + if (entry && Date.now() >= entry.expiresAt) { + seen.delete(k); + return undefined; + } + return entry; +} + +/** Record a reply of `key` that reasoned; `asked`: its request asked to. */ +export function noteReasoningSeen( + key: ReasoningDialectKey, + asked: boolean, +): void { + const k = cacheKey(key); + seen.set(k, { + unasked: !asked || (seenEntry(k)?.unasked ?? false), + expiresAt: Date.now() + DIALECT_TTL_MS, + }); +} + +/** + * True when `key` was seen reasoning in a way that speaks for a request that + * did (`asked`) or did not ask to reason: a model that reasoned unasked + * speaks for both, one seen reasoning only when asked for the first. + */ +export function reasoningSeen( + key: ReasoningDialectKey, + asked: boolean, +): boolean { + const entry = seenEntry(cacheKey(key)); + return entry !== undefined && (entry.unasked || asked); +} + /** Test hook: forget everything learned. */ export function resetReasoningDialectCache(): void { learned.clear(); + seen.clear(); } /** diff --git a/src/lib/ai/reasoning/support.ts b/src/lib/ai/reasoning/support.ts index 38f41989c..b5555ab00 100644 --- a/src/lib/ai/reasoning/support.ts +++ b/src/lib/ai/reasoning/support.ts @@ -275,16 +275,25 @@ export function reasoningTitleOf(text: string): string | null { * * #1276 — a reply can also carry only the closing tag: GLM, Qwen3 and * DeepSeek-R1 chat templates end the generation prompt with ``, so the - * model's output starts inside the block. Everything before the first - * `` of such a reply is reasoning. + * model's output starts inside the block. Everything before that `` + * is reasoning when the tag opens a line (the templates close the block on a + * line of its own, or at the very start), or anywhere once the model is known + * to reason (`knownReasoner`). An ordinary answer that mentions the tag in a + * sentence is left whole. */ -export function splitThinkTags(content: string): { +export function splitThinkTags( + content: string, + knownReasoner = false, +): { content: string; reasoning: string | null; } { const m = /^\s*([\s\S]*?)(<\/think>|$)/.exec(content) ?? - /^([\s\S]*?)(<\/think>)/.exec(content); + (knownReasoner + ? /^([\s\S]*?)(<\/think>)/ + : /^((?:[\s\S]*?\n)?)[ \t]*(<\/think>)/ + ).exec(content); if (!m) return { content, reasoning: null }; const reasoning = m[1].trim(); const rest = m[2] ? content.slice(m[0].length).replace(/^\s+/, "") : ""; @@ -299,10 +308,12 @@ export function splitThinkTags(content: string): { * off by {@link splitThinkTags}. * - A reply that ran into a ceiling which held room for reasoning * (`finish_reason: "length"`, `reasoningRoom`), with no reasoning field - * beside it, no tag and no tool call, is reasoning the server did not - * separate: the budget was sized for thinking plus an answer, so text - * that fills all of it unlabelled is the thought, cut off. It is never - * the answer. + * beside it, no tag and no tool call, from a model known to reason + * (`knownReasoner`), is reasoning the server did not separate: the budget + * was sized for thinking plus an answer, so text that fills all of it + * unlabelled is the thought, cut off. From any other model it is the + * answer, cut off, and the caller reads `finish_reason` (a model that + * does not reason, asked with any level, still answers). * * `reasoning` is the text taken out of `content`, for the reasoning channel. * `unseparated` marks the second case so the caller can say so in its log. @@ -316,11 +327,13 @@ export function answerOf( hasToolCalls: boolean; /** The request's ceiling held room for reasoning on top of the answer. */ reasoningRoom: boolean; + /** This endpoint and model have shown they reason (`knownReasoner`). */ + knownReasoner: boolean; }, ): { content: string; reasoning: string | null; unseparated: boolean } { const text = content ?? ""; if (!text) return { content: "", reasoning: null, unseparated: false }; - const split = splitThinkTags(text); + const split = splitThinkTags(text, reply.knownReasoner); if (split.content !== text || split.reasoning !== null) { return { content: split.content, @@ -331,6 +344,7 @@ export function answerOf( if ( reply.finishReason === "length" && reply.reasoningRoom && + reply.knownReasoner && !reply.fieldedReasoning && !reply.hasToolCalls ) {