From 685cb2fa01ff4b816fb6a9640d426b946e3c4ffb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Fri, 30 Oct 2026 17:13:09 +0100 Subject: [PATCH] fix(ai): learn the max_tokens refusal from the field the request carried A call that ran beside the one that learned the lesson read today's selection instead and failed with the 400. The matcher now also wants the refusal to name max_tokens, so a refusal of another field that echoes the request does not switch the token field. Refs #1304 --- .../ai/__tests__/azure-token-budget.test.ts | 29 +++++++++++++++++++ src/lib/ai/openai-capabilities.ts | 9 ++++-- src/lib/ai/openai-client.ts | 27 ++++++++--------- 3 files changed, 49 insertions(+), 16 deletions(-) diff --git a/src/lib/ai/__tests__/azure-token-budget.test.ts b/src/lib/ai/__tests__/azure-token-budget.test.ts index 685513d85..8bca021ac 100644 --- a/src/lib/ai/__tests__/azure-token-budget.test.ts +++ b/src/lib/ai/__tests__/azure-token-budget.test.ts @@ -197,11 +197,40 @@ describe("an endpoint that refuses max_tokens", () => { expect(mock).toHaveBeenCalledTimes(1); }); + it("asks again on every call that sent max_tokens, also beside one that learned it", async () => { + const mock = vi + .fn() + .mockResolvedValueOnce(refused(400, AZURE_REFUSAL)) + .mockResolvedValueOnce(refused(400, AZURE_REFUSAL)) + .mockResolvedValue(ok()); + vi.stubGlobal("fetch", mock); + const client = gateway("prod-chat"); + const results = await Promise.allSettled([ + client.generateCompletion(turn()), + client.generateCompletion(turn()), + ]); + expect(results.map((r) => r.status)).toEqual(["fulfilled", "fulfilled"]); + expect(mock).toHaveBeenCalledTimes(4); + }); + it("reads only a 400 that names the field", () => { expect(isMaxTokensRejection(400, AZURE_REFUSAL)).toBe(true); expect(isMaxTokensRejection(422, AZURE_REFUSAL)).toBe(false); expect( isMaxTokensRejection(400, '{"error":"Unsupported parameter: top_k"}'), ).toBe(false); + // A refusal of another field that echoes the request. + expect( + isMaxTokensRejection( + 400, + '{"error":"Unsupported parameter: reasoning_effort is not supported","request":{"max_tokens":1000}}', + ), + ).toBe(false); + expect( + isMaxTokensRejection( + 400, + '{"error":"max_tokens is not supported with this model"}', + ), + ).toBe(true); }); }); diff --git a/src/lib/ai/openai-capabilities.ts b/src/lib/ai/openai-capabilities.ts index 6848722ee..3ff7014d8 100644 --- a/src/lib/ai/openai-capabilities.ts +++ b/src/lib/ai/openai-capabilities.ts @@ -99,14 +99,17 @@ export function resetMaxTokensRefusals(): void { /** * True when a 400 says the model takes `max_completion_tokens` and not - * `max_tokens`. A complaint about the value ("max_tokens is too large") is - * not a refusal of the field and does not match. + * `max_tokens`: it names the replacement, or it refuses `max_tokens` by + * name. A complaint about the value ("max_tokens is too large") or about + * another field that echoes the request is not a refusal of the field. */ export function isMaxTokensRejection(status: number, body: string): boolean { if (status !== 400 || !/\bmax_tokens\b/.test(body)) return false; return ( /max_completion_tokens/.test(body) || - /unsupported[\s_-]*parameter|not supported/i.test(body) + /unsupported[\s_-]*parameter\W{0,4}max_tokens\b|\bmax_tokens\W{0,4}(is\s+)?(not\s+supported|unsupported)/i.test( + body, + ) ); } diff --git a/src/lib/ai/openai-client.ts b/src/lib/ai/openai-client.ts index c4430706a..9bdb33623 100644 --- a/src/lib/ai/openai-client.ts +++ b/src/lib/ai/openai-client.ts @@ -206,11 +206,11 @@ export class OpenAIClient implements AIProvider { ? jsonModeDialectFor(this.config.baseUrl) : "response_format"; const sendJsonFormat = useJsonFormat && jsonDialect === "response_format"; - const capabilities = selectOpenAIChatCompletionsCapabilities( + const caps = selectOpenAIChatCompletionsCapabilities( this.config.baseUrl, this.config.model, ); - const reasoningWire = this.planReasoning(params, hasTools, capabilities); + const reasoningWire = this.planReasoning(params, hasTools, caps); // #1126 — the gateway entry's own setting, for a call without a level, // unless the wire carries one or the endpoint refused the field (#1279). const entryEffort = @@ -225,13 +225,13 @@ export class OpenAIClient implements AIProvider { reasoningWire.kind !== "none" ? reasoningWire.effort : entryEffort; const thinkingTokens = this.thinkingRoom(params, reasoningWire); const tokenBudget = (params.maxTokens ?? 1000) + thinkingTokens; - const capabilityParams = capabilities.supportsSamplingControls + const capabilityParams = caps.supportsSamplingControls ? { - [capabilities.tokenBudgetField]: tokenBudget, + [caps.tokenBudgetField]: tokenBudget, temperature: params.temperature ?? 0.3, ...(params.seed !== undefined ? { seed: params.seed } : {}), } - : { [capabilities.tokenBudgetField]: tokenBudget }; + : { [caps.tokenBudgetField]: tokenBudget }; // A gateway that needs no bearer (a LAN LiteLLM without a master key) // gets no `Authorization` header at all rather than an empty one — an @@ -299,7 +299,7 @@ export class OpenAIClient implements AIProvider { ); if (!res.ok) { - return this.failed(res, params, hops, { sent, sendJsonFormat }); + return this.failed(res, params, hops, { sent, sendJsonFormat, caps }); } // v1.33.1 (#470) — the standard `response_format` request succeeded: pin @@ -512,7 +512,11 @@ export class OpenAIClient implements AIProvider { res: Response, params: CompletionParams, hops: RequestHops, - sent: { sent: string | null; sendJsonFormat: boolean }, + sent: { + sent: string | null; + sendJsonFormat: boolean; + caps: OpenAIChatCompletionsCapabilities; + }, ): Promise { const bodyExcerpt = sanitiseBodyExcerpt(await res.text().catch(() => "")); // The entry's own setting (#1126) included: a refused field is learned @@ -535,13 +539,10 @@ export class OpenAIClient implements AIProvider { this.isGateway && sent.sendJsonFormat ? { endpoint: this.config.baseUrl, action: "ai.compat.jsonDialect" } : null, - // The request that failed carried `max_tokens`: nothing is learned - // between sending it and reading its refusal, so the selection holds. + // The field this request carried, not today's selection: a call + // running beside it may have learned the lesson meanwhile. budget: - selectOpenAIChatCompletionsCapabilities( - this.config.baseUrl, - this.config.model, - ).tokenBudgetField === "max_tokens" + sent.caps.tokenBudgetField === "max_tokens" ? { endpoint: this.config.baseUrl } : null, });