From 7b0dcbf560a917c3334531be9c857cfbd0752a32 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Mon, 22 Jun 2026 08:11:16 +0200 Subject: [PATCH] fix(ai): gate OpenAI JSON mode on responseFormat instead of forcing it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The OpenAI client pinned `response_format: { type: "json_object" }` on every call. The Coach tool loop's forced-final round runs with `tool_choice: "none"` and expects plain prose, but JSON mode coerced the reply into a JSON object — the user saw a JSON blob or an empty reply instead of the coach's answer. Send `response_format` only when the caller opts in with `responseFormat: "json"` and no tools are present, mirroring the Anthropic client's prefill gate. The JSON-consuming callers that relied on the implicit mode (insight generate + briefing retry, medication extraction, the connection-test probe) now pass the flag explicitly; the prose Coach path and every tool round stay out of JSON mode. Codex already omitted the flag and is unaffected. --- src/app/api/ai/test/route.ts | 3 ++ src/app/api/insights/generate/route.ts | 5 ++ src/app/api/medications/extract/route.ts | 3 ++ src/lib/ai/__tests__/openai-client.test.ts | 61 ++++++++++++++++++++++ src/lib/ai/openai-client.ts | 19 +++++-- src/lib/ai/types.ts | 9 ++-- 6 files changed, 91 insertions(+), 9 deletions(-) diff --git a/src/app/api/ai/test/route.ts b/src/app/api/ai/test/route.ts index 984b75e13..aae4bbcb6 100644 --- a/src/app/api/ai/test/route.ts +++ b/src/app/api/ai/test/route.ts @@ -72,6 +72,9 @@ export const POST = apiHandler(async (request: NextRequest) => { user: 'Reply with the JSON object {"ok": true} and nothing else.', temperature: 0, maxTokens: 32, + // The probe asks for a JSON object — keep the OpenAI / Codex strict + // JSON mode it relied on before the response_format gate landed. + responseFormat: "json", }), ); diff --git a/src/app/api/insights/generate/route.ts b/src/app/api/insights/generate/route.ts index 134fac3cc..a27c05e75 100644 --- a/src/app/api/insights/generate/route.ts +++ b/src/app/api/insights/generate/route.ts @@ -682,6 +682,9 @@ export const POST = apiHandler((request: NextRequest) => user: userPrompt, temperature: 0.3, maxTokens: 1500, + // The reply is parsed with `JSON.parse` below, so opt the OpenAI / + // Codex chains into their strict JSON mode (gated on this flag). + responseFormat: "json", }), }); result = fallback.result; @@ -815,6 +818,8 @@ export const POST = apiHandler((request: NextRequest) => user: `${userPrompt}\n\n${buildBriefingGroundingCorrection(ungrounded)}`, temperature: 0.3, maxTokens: 1500, + // Parsed with `JSON.parse` below — opt OpenAI / Codex into JSON mode. + responseFormat: "json", }), }); const parsedRetry = JSON.parse(retry.result.content); diff --git a/src/app/api/medications/extract/route.ts b/src/app/api/medications/extract/route.ts index 954406251..4119ee4bc 100644 --- a/src/app/api/medications/extract/route.ts +++ b/src/app/api/medications/extract/route.ts @@ -195,6 +195,9 @@ async function handleExtract(request: NextRequest): Promise { // The reply body is a single small JSON object; 600 is well // over the empirical max we have seen on a 2k-char input. maxTokens: 600, + // Parsed with `parseModelJson` (JSON.parse) below — opt the OpenAI / + // Codex chains into their strict JSON-object mode (gated on this flag). + responseFormat: "json", }), }); } catch (err) { diff --git a/src/lib/ai/__tests__/openai-client.test.ts b/src/lib/ai/__tests__/openai-client.test.ts index 25f867a98..170223801 100644 --- a/src/lib/ai/__tests__/openai-client.test.ts +++ b/src/lib/ai/__tests__/openai-client.test.ts @@ -28,6 +28,7 @@ describe("OpenAIClient", () => { singleUserTurn({ system: "You are a doctor.", user: "Analyze this data.", + responseFormat: "json", }), ); @@ -46,6 +47,66 @@ describe("OpenAIClient", () => { expect(body).not.toHaveProperty("seed"); }); + it("forces JSON mode only when responseFormat is json AND no tools (M-1)", async () => { + const replyContent = (content: string) => + vi.fn().mockResolvedValue({ + ok: true, + json: () => + Promise.resolve({ + choices: [{ message: { content } }], + usage: { total_tokens: 1 }, + }), + }); + + const client = new OpenAIClient({ + apiKey: "sk-test", + model: "gpt-4o-mini", + baseUrl: "https://api.openai.com/v1", + }); + + // 1. Prose Coach path (no responseFormat, no tools) → NO json_object. This + // is the F1 forced-final round contract: JSON mode would corrupt prose. + const proseFetch = replyContent("Here is a plain-prose reply."); + vi.stubGlobal("fetch", proseFetch); + await client.generateCompletion( + singleUserTurn({ system: "coach", user: "how am I doing?" }), + ); + expect(JSON.parse(proseFetch.mock.calls[0][1].body)).not.toHaveProperty( + "response_format", + ); + + // 2. responseFormat:"json" but tools present → still NO json_object (a tool + // round must not be coerced into a JSON object). + const toolFetch = replyContent("{}"); + vi.stubGlobal("fetch", toolFetch); + await client.generateCompletion({ + system: "s", + messages: [{ role: "user", content: "u" }], + responseFormat: "json", + tools: [ + { + name: "t", + description: "d", + parameters: { type: "object", properties: {} }, + }, + ], + }); + expect(JSON.parse(toolFetch.mock.calls[0][1].body)).not.toHaveProperty( + "response_format", + ); + + // 3. responseFormat:"json" with no tools → json_object IS sent (insight / + // extraction callers keep strict JSON). + const jsonFetch = replyContent('{"summary":"x"}'); + vi.stubGlobal("fetch", jsonFetch); + await client.generateCompletion( + singleUserTurn({ system: "s", user: "u", responseFormat: "json" }), + ); + expect(JSON.parse(jsonFetch.mock.calls[0][1].body).response_format).toEqual( + { type: "json_object" }, + ); + }); + it("threads a deterministic seed onto the request body when supplied", async () => { const mockFetch = vi.fn().mockResolvedValue({ ok: true, diff --git a/src/lib/ai/openai-client.ts b/src/lib/ai/openai-client.ts index d55a58c81..e4a8c808d 100644 --- a/src/lib/ai/openai-client.ts +++ b/src/lib/ai/openai-client.ts @@ -49,10 +49,17 @@ export class OpenAIClient implements AIProvider { // v1.20.0 — tool plumbing. The defs map onto the OpenAI `function` tool // wire; F1 supplies real defs and consumes the parsed `toolCalls`. No F4 // call site sets `tools`, so the text-only body is unchanged. - const tools = - params.tools && params.tools.length > 0 - ? buildOpenAITools(params.tools) - : undefined; + const hasTools = !!params.tools && params.tools.length > 0; + const tools = hasTools ? buildOpenAITools(params.tools!) : undefined; + + // Only force OpenAI's strict JSON mode when the caller actually consumes a + // JSON object AND no tools are in play — mirrors the Anthropic client's + // `usePrefill` gate. JSON mode coerces `message.content` into a valid JSON + // object, which contradicts the Coach prose contract; the F1 tool loop's + // forced-final round (toolChoice:"none", no tools, no responseFormat) must + // therefore stay out of JSON mode, and every tool round is non-JSON by + // construction. Insight/extraction callers opt in with `responseFormat:"json"`. + const useJsonFormat = params.responseFormat === "json" && !hasTools; const res = await safeFetch( url, @@ -67,7 +74,9 @@ export class OpenAIClient implements AIProvider { messages, temperature: params.temperature ?? 0.3, max_tokens: params.maxTokens ?? 1000, - response_format: { type: "json_object" }, + ...(useJsonFormat + ? { response_format: { type: "json_object" } } + : {}), // Deterministic seed for reproducible reference output; omitted // (undefined → dropped by JSON.stringify) when the caller does // not pin one (e.g. the seedless daily-briefing re-roll). diff --git a/src/lib/ai/types.ts b/src/lib/ai/types.ts index 50612402d..0e1cc8028 100644 --- a/src/lib/ai/types.ts +++ b/src/lib/ai/types.ts @@ -246,13 +246,14 @@ export interface CompletionParams { seed?: number; /** * v1.18.7 — opt the structured (JSON) surfaces into the provider's - * strongest JSON-reliability mode. When `"json"`: + * strongest JSON-reliability mode. When `"json"` (and no `tools`): * - local (Ollama / OpenAI-compatible): adds `format: "json"`. * - Anthropic: prefills the assistant turn with `{` so the first token * is forced into a JSON object. - * The OpenAI client already pins `response_format: json_object` - * unconditionally and is unaffected. The Coach (prose, NOT JSON) leaves - * this unset, so its streaming path is untouched. + * - OpenAI / Codex: sends `response_format: { type: "json_object" }`. + * The Coach (prose, NOT JSON) leaves this unset, so no provider coerces + * its reply into a JSON object — including the OpenAI client, which now + * gates `response_format` on this flag rather than pinning it always. * * Mutually exclusive with `tools` on Anthropic: the `{`-prefill injects an * assistant turn that collides with a `tool_use` response, so the Anthropic