From 21af34405452e9bd6e5b8778aa0ce206d171416f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Fri, 30 Oct 2026 17:08:43 +0100 Subject: [PATCH] fix(ai): send max_completion_tokens to Azure OpenAI reasoning deployments An OpenAI-compatible gateway entry pointed at an Azure OpenAI v1 endpoint was treated as a generic gateway and sent max_tokens and temperature, which a reasoning deployment refuses with a 400. An Azure v1 endpoint now takes the api.openai.com contract for a reasoning model, and any gateway that refuses max_tokens for a model is asked again with max_completion_tokens and remembers it, which covers deployments named something of their own. Refs #1304 --- docs/integrations/ai-providers.md | 12 + .../ai/__tests__/azure-token-budget.test.ts | 207 ++++++++++++++++++ src/lib/ai/json-dialect.ts | 6 +- src/lib/ai/local-client.ts | 2 + src/lib/ai/openai-capabilities.ts | 74 ++++++- src/lib/ai/openai-client.ts | 12 +- src/lib/ai/wire-refusal.ts | 28 ++- 7 files changed, 329 insertions(+), 12 deletions(-) create mode 100644 src/lib/ai/__tests__/azure-token-budget.test.ts diff --git a/docs/integrations/ai-providers.md b/docs/integrations/ai-providers.md index 84e9d20f0..f099f6fb6 100644 --- a/docs/integrations/ai-providers.md +++ b/docs/integrations/ai-providers.md @@ -216,6 +216,18 @@ JSON surfaces send the standard the field is detected on the first refusal and retried without it, so strict gateways and older local servers both work unmodified. +**Azure OpenAI.** Point the gateway at the resource's v1 endpoint, +`https://.openai.azure.com/openai/v1`, and put the +deployment name in Model. A reasoning model (gpt-5 on, the o-series) +takes `max_completion_tokens` and no `temperature`, the same as on +`api.openai.com`. HealthLog sends that from the first request when the +deployment is named after its model (`gpt-6-luna-eu`). A deployment +with a name of its own (`prod-chat`) gets the usual `max_tokens` +once; Azure refuses it, and the request goes out again with +`max_completion_tokens`. HealthLog remembers that for the endpoint and +deployment until the container restarts. Any other gateway that +refuses `max_tokens` the same way is handled the same way. + **When the gateway refuses.** OpenRouter answers a refused generation with HTTP 200 and the reason inside the body, so every refusal used to reach the provider health card as an empty reply with no status. It now diff --git a/src/lib/ai/__tests__/azure-token-budget.test.ts b/src/lib/ai/__tests__/azure-token-budget.test.ts new file mode 100644 index 000000000..685513d85 --- /dev/null +++ b/src/lib/ai/__tests__/azure-token-budget.test.ts @@ -0,0 +1,207 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; + +import { resetJsonModeDialectCache } from "../json-dialect"; +import { OpenAIClient } from "../openai-client"; +import { + isAzureOpenAIV1Endpoint, + isMaxTokensRejection, + resetMaxTokensRefusals, + selectOpenAIChatCompletionsCapabilities, +} from "../openai-capabilities"; +import { resetReasoningDialectCache } from "../reasoning/dialect-cache"; +import { singleUserTurn } from "../types"; + +/** + * #1304 — an OpenAI-compatible gateway entry pointed straight at an Azure + * OpenAI v1 endpoint. A reasoning deployment refuses `max_tokens` with a 400; + * the request has to carry `max_completion_tokens` and no sampling controls, + * as it does on `api.openai.com`. Every other gateway keeps the legacy wire. + */ + +vi.mock("undici", async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + fetch: (input: unknown, init?: unknown) => + (globalThis.fetch as unknown as (i: unknown, n?: unknown) => unknown)( + input, + init, + ), + }; +}); + +vi.mock("@/lib/logging/context", () => ({ annotate: vi.fn() })); + +const AZURE = "https://example-resource.openai.azure.com/openai/v1"; + +// The upstream error the report quotes, verbatim. +const AZURE_REFUSAL = JSON.stringify({ + error: { + message: + "Unsupported parameter: 'max_tokens' is not supported with this model. Use 'max_completion_tokens' instead.", + type: "invalid_request_error", + param: "max_tokens", + code: "unsupported_parameter", + }, +}); + +function ok() { + return { + ok: true, + json: () => + Promise.resolve({ + choices: [ + { message: { content: '{"ok":true}' }, finish_reason: "stop" }, + ], + usage: { total_tokens: 5 }, + }), + }; +} + +function refused(status: number, body: string) { + return { ok: false, status, text: () => Promise.resolve(body) }; +} + +function gateway(model: string, baseUrl = AZURE) { + return new OpenAIClient({ + apiKey: "k", + model, + baseUrl, + providerType: "openai-compatible", + }); +} + +const turn = (responseFormat?: "json") => + singleUserTurn({ system: "s", user: "u", responseFormat }); + +function bodyOf(mock: ReturnType, call = 0) { + return JSON.parse(mock.mock.calls[call][1].body); +} + +beforeEach(() => { + vi.restoreAllMocks(); + resetJsonModeDialectCache(); + resetReasoningDialectCache(); + resetMaxTokensRefusals(); +}); + +describe("Azure OpenAI v1 endpoint", () => { + it.each([ + AZURE, + `${AZURE}/`, + "https://example-resource.cognitiveservices.azure.com/openai/v1", + "https://example-resource.services.ai.azure.com/openai/v1", + ])("recognises %s", (url) => { + expect(isAzureOpenAIV1Endpoint(url)).toBe(true); + }); + + it.each([ + "http://example-resource.openai.azure.com/openai/v1", + "https://example-resource.openai.azure.com.example.com/openai/v1", + "https://openai.azure.com/openai/v1", + "https://example-resource.openai.azure.com/openai/deployments/x", + "https://example-resource.openai.azure.com:8443/openai/v1", + `${AZURE}?api-version=preview`, + "https://litellm.example.com/v1", + "not a url", + ])("does not take %s for one", (url) => { + expect(isAzureOpenAIV1Endpoint(url)).toBe(false); + }); + + it("sends a reasoning deployment max_completion_tokens and no temperature", async () => { + const mock = vi.fn().mockResolvedValue(ok()); + vi.stubGlobal("fetch", mock); + await gateway("gpt-6-luna-example").generateCompletion(turn()); + expect(mock).toHaveBeenCalledTimes(1); + const body = bodyOf(mock); + expect(body.max_completion_tokens).toBeGreaterThan(0); + expect(body).not.toHaveProperty("max_tokens"); + expect(body).not.toHaveProperty("temperature"); + expect(body).not.toHaveProperty("seed"); + }); + + it("keeps the legacy wire for a GPT-4 deployment", async () => { + const mock = vi.fn().mockResolvedValue(ok()); + vi.stubGlobal("fetch", mock); + await gateway("gpt-4o").generateCompletion(turn()); + const body = bodyOf(mock); + expect(body.max_tokens).toBeGreaterThan(0); + expect(body.temperature).toBe(0.3); + expect(body).not.toHaveProperty("max_completion_tokens"); + }); + + it("keeps the legacy wire for a reasoning model name behind another gateway", () => { + expect( + selectOpenAIChatCompletionsCapabilities( + "https://litellm.example.com/v1", + "gpt-6-luna", + ).tokenBudgetField, + ).toBe("max_tokens"); + }); +}); + +describe("an endpoint that refuses max_tokens", () => { + it("asks again with max_completion_tokens for an opaque deployment name and remembers it", async () => { + const mock = vi + .fn() + .mockResolvedValueOnce(refused(400, AZURE_REFUSAL)) + .mockResolvedValue(ok()); + vi.stubGlobal("fetch", mock); + const client = gateway("prod-chat"); + + const result = await client.generateCompletion(turn()); + expect(result.content).toBe('{"ok":true}'); + expect(mock).toHaveBeenCalledTimes(2); + expect(bodyOf(mock, 0).max_tokens).toBeGreaterThan(0); + const repeat = bodyOf(mock, 1); + expect(repeat.max_completion_tokens).toBe(bodyOf(mock, 0).max_tokens); + expect(repeat).not.toHaveProperty("max_tokens"); + expect(repeat).not.toHaveProperty("temperature"); + + await client.generateCompletion(turn()); + expect(mock).toHaveBeenCalledTimes(3); + expect(bodyOf(mock, 2)).toHaveProperty("max_completion_tokens"); + }); + + it("learns it for that model only", async () => { + const mock = vi + .fn() + .mockResolvedValueOnce(refused(400, AZURE_REFUSAL)) + .mockResolvedValue(ok()); + vi.stubGlobal("fetch", mock); + await gateway("prod-chat").generateCompletion(turn()); + await gateway("prod-legacy").generateCompletion(turn()); + expect(bodyOf(mock, 2)).toHaveProperty("max_tokens"); + }); + + it("keeps JSON mode on the repeat", async () => { + const mock = vi + .fn() + .mockResolvedValueOnce(refused(400, AZURE_REFUSAL)) + .mockResolvedValue(ok()); + vi.stubGlobal("fetch", mock); + await gateway("prod-chat").generateCompletion(turn("json")); + expect(bodyOf(mock, 1).response_format).toEqual({ type: "json_object" }); + }); + + it("does not take a complaint about the value for a refusal of the field", async () => { + const mock = vi + .fn() + .mockResolvedValue( + refused(400, '{"error":"max_tokens is too large: 900000"}'), + ); + vi.stubGlobal("fetch", mock); + await expect( + gateway("prod-chat").generateCompletion(turn()), + ).rejects.toMatchObject({ httpStatus: 400 }); + expect(mock).toHaveBeenCalledTimes(1); + }); + + it("reads only a 400 that names the field", () => { + expect(isMaxTokensRejection(400, AZURE_REFUSAL)).toBe(true); + expect(isMaxTokensRejection(422, AZURE_REFUSAL)).toBe(false); + expect( + isMaxTokensRejection(400, '{"error":"Unsupported parameter: top_k"}'), + ).toBe(false); + }); +}); diff --git a/src/lib/ai/json-dialect.ts b/src/lib/ai/json-dialect.ts index 0c1859105..5442707eb 100644 --- a/src/lib/ai/json-dialect.ts +++ b/src/lib/ai/json-dialect.ts @@ -57,8 +57,8 @@ export function resetJsonModeDialectCache(): void { * True when a 4xx error body reads as "this endpoint rejects the * `response_format` field" — either it names the field outright or it * complains about an unknown/unexpected parameter. A complaint that names - * `reasoning_effort` and not `response_format` is about the reasoning field, - * so it must not switch the endpoint's JSON mode off. + * `reasoning_effort` or `max_tokens` (#1304) and not `response_format` is + * about that field, so it must not switch the endpoint's JSON mode off. */ export function isResponseFormatRejection( status: number, @@ -66,7 +66,7 @@ export function isResponseFormatRejection( ): boolean { if (status < 400 || status >= 500) return false; if ( - /reasoning[\s_-]*effort/i.test(bodyExcerpt) && + /reasoning[\s_-]*effort|\bmax_tokens\b/i.test(bodyExcerpt) && !/response_format/i.test(bodyExcerpt) ) { return false; diff --git a/src/lib/ai/local-client.ts b/src/lib/ai/local-client.ts index 4fbe52d6b..bf97f551a 100644 --- a/src/lib/ai/local-client.ts +++ b/src/lib/ai/local-client.ts @@ -334,6 +334,8 @@ export class LocalOpenAICompatibleClient implements AIProvider { json: sent.json ? { endpoint: this.config.baseUrl, action: "ai.local.jsonDialect" } : null, + // Local runtimes take `max_tokens`; this wire has no other field. + budget: null, }); if ((lesson || sent.streamed) && hops.canSend) { return this.generateCompletion(params, hops); diff --git a/src/lib/ai/openai-capabilities.ts b/src/lib/ai/openai-capabilities.ts index 407b6c2e4..6848722ee 100644 --- a/src/lib/ai/openai-capabilities.ts +++ b/src/lib/ai/openai-capabilities.ts @@ -47,16 +47,86 @@ export function isCanonicalOpenAIEndpoint(baseUrl: string): boolean { } } +/** The hosts an Azure OpenAI resource is served from. */ +const AZURE_OPENAI_HOST = + /^[a-z0-9][a-z0-9-]*\.(openai\.azure\.com|cognitiveservices\.azure\.com|services\.ai\.azure\.com)$/; + +/** + * #1304 — an Azure OpenAI resource's v1 endpoint + * (`https://.openai.azure.com/openai/v1`). It speaks the same + * request contract as `api.openai.com` for the same model, but the model + * field carries the deployment name, which only identifies the model when + * the deployment was named after it. An opaque alias is caught by the + * refusal it draws instead (`rememberMaxTokensRefused`). + */ +export function isAzureOpenAIV1Endpoint(baseUrl: string): boolean { + try { + const url = new URL(baseUrl); + return ( + url.protocol === "https:" && + AZURE_OPENAI_HOST.test(url.hostname) && + url.port === "" && + url.pathname.replace(/\/+$/, "") === "/openai/v1" && + url.username === "" && + url.password === "" && + url.search === "" && + url.hash === "" + ); + } catch { + return false; + } +} + +/** + * #1304 — endpoint and model pairs that refused `max_tokens`, for the + * process lifetime. Keyed by both, because one gateway can front a legacy + * model and a reasoning model side by side. + */ +const maxTokensRefused = new Set(); + +function refusalKey(baseUrl: string, model: string): string { + return `${baseUrl}\n${model}`; +} + +export function rememberMaxTokensRefused(baseUrl: string, model: string): void { + maxTokensRefused.add(refusalKey(baseUrl, model)); +} + +/** Test hook — clears the learned `max_tokens` refusals. */ +export function resetMaxTokensRefusals(): void { + maxTokensRefused.clear(); +} + +/** + * True when a 400 says the model takes `max_completion_tokens` and not + * `max_tokens`. A complaint about the value ("max_tokens is too large") is + * not a refusal of the field and does not match. + */ +export function isMaxTokensRejection(status: number, body: string): boolean { + if (status !== 400 || !/\bmax_tokens\b/.test(body)) return false; + return ( + /max_completion_tokens/.test(body) || + /unsupported[\s_-]*parameter|not supported/i.test(body) + ); +} + /** * Select Chat Completions request capabilities from both endpoint and model. * Custom OpenAI-compatible gateways retain the broadly supported legacy wire, - * even when their model name resembles a modern OpenAI model. + * even when their model name resembles a modern OpenAI model, until the + * endpoint refuses `max_tokens` for that model. `api.openai.com` and an + * Azure OpenAI v1 endpoint take the modern wire for a reasoning model. */ export function selectOpenAIChatCompletionsCapabilities( baseUrl: string, model: string, ): OpenAIChatCompletionsCapabilities { - return isCanonicalOpenAIEndpoint(baseUrl) && isModernCanonicalModel(model) + if (maxTokensRefused.has(refusalKey(baseUrl, model))) { + return MODERN_CANONICAL_CAPABILITIES; + } + return (isCanonicalOpenAIEndpoint(baseUrl) || + isAzureOpenAIV1Endpoint(baseUrl)) && + isModernCanonicalModel(model) ? MODERN_CANONICAL_CAPABILITIES : LEGACY_COMPATIBLE_CAPABILITIES; } diff --git a/src/lib/ai/openai-client.ts b/src/lib/ai/openai-client.ts index 1e144ac2c..c4430706a 100644 --- a/src/lib/ai/openai-client.ts +++ b/src/lib/ai/openai-client.ts @@ -498,7 +498,8 @@ export class OpenAIClient implements AIProvider { * structured field (with anything key-shaped stripped), so an upstream * incident is diagnosable from the log instead of an opaque status. * - * An endpoint or model that refuses the reasoning field, or (#470, gateway + * An endpoint or model that refuses the reasoning field, `max_tokens` + * (#1304, asked again with `max_completion_tokens`), or (#470, gateway * only) the JSON flag, is asked once more without it and remembers * (`wire-refusal.ts`), inside the call's request budget. A refusal the * budget leaves no repeat for is marked `reasoningRefused`. @@ -534,6 +535,15 @@ export class OpenAIClient implements AIProvider { this.isGateway && sent.sendJsonFormat ? { endpoint: this.config.baseUrl, action: "ai.compat.jsonDialect" } : null, + // The request that failed carried `max_tokens`: nothing is learned + // between sending it and reading its refusal, so the selection holds. + budget: + selectOpenAIChatCompletionsCapabilities( + this.config.baseUrl, + this.config.model, + ).tokenBudgetField === "max_tokens" + ? { endpoint: this.config.baseUrl } + : null, }); if (lesson && hops.canSend) return this.generateCompletion(params, hops); const modelNotServed = await this.modelNoLongerServed(res.status); diff --git a/src/lib/ai/wire-refusal.ts b/src/lib/ai/wire-refusal.ts index 64136db6b..a1909722d 100644 --- a/src/lib/ai/wire-refusal.ts +++ b/src/lib/ai/wire-refusal.ts @@ -3,10 +3,12 @@ * clients, in one place for both. * * Only an HTTP 400 that names a field the request carried teaches anything: - * the reasoning field (`isReasoningParameterRejection`, learned per endpoint - * and model) or the JSON flag (`isResponseFormatRejection`, learned per - * endpoint). The reasoning field is checked first, so a body that names it - * never switches JSON mode off. Whether the call then sends the lesser request + * `max_tokens` where the model takes `max_completion_tokens` (#1304, + * `isMaxTokensRejection`, learned per endpoint and model), the reasoning + * field (`isReasoningParameterRejection`, learned per endpoint and model) or + * the JSON flag (`isResponseFormatRejection`, learned per endpoint). The + * JSON flag is checked last, so a body that names another field never + * switches JSON mode off. Whether the call then sends the lesser request * is the client's, inside its request budget (`request-hops.ts`). * * Server-only. @@ -17,13 +19,17 @@ import { isResponseFormatRejection, rememberJsonModeDialect, } from "./json-dialect"; +import { + isMaxTokensRejection, + rememberMaxTokensRefused, +} from "./openai-capabilities"; import { learnReasoningFieldRefused } from "./reasoning/answerless"; import { isReasoningParameterRejection, type ReasoningDialectKey, } from "./reasoning/dialect-cache"; -export type RefusalLesson = "reasoning" | "json" | null; +export type RefusalLesson = "budget" | "reasoning" | "json" | null; export function learnFromRefusal(args: { status: number; @@ -39,8 +45,18 @@ export function learnFromRefusal(args: { } | null; /** The endpoint, when the request carried the JSON flag. */ json: { endpoint: string; action: string } | null; + /** The endpoint, when the request carried `max_tokens` it could drop. */ + budget: { endpoint: string } | null; }): RefusalLesson { - const { reasoning, json } = args; + const { reasoning, json, budget } = args; + if (budget && isMaxTokensRejection(args.status, args.body)) { + rememberMaxTokensRefused(budget.endpoint, args.model); + annotate({ + action: { name: "ai.compat.tokenBudgetField" }, + meta: { field: "max_completion_tokens", httpStatus: args.status }, + }); + return "budget"; + } if ( reasoning && isReasoningParameterRejection(