From 178d4d761072fdbfdfd3ed59ea845fdcfbbdbfb4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Sat, 20 Jun 2026 18:59:37 +0200 Subject: [PATCH] feat(labs): unlock native Codex vision for lab OCR MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Image input rides the ChatGPT-OAuth codex/responses endpoint via the input_image content block (protocol spec 2b) — no vision API key. The codex client folds params.images into input_image blocks alongside the input_text; text-only calls keep the prior byte-identical wire. The vision gate now resolves codex capability against the working slug (GPT-5.x / gpt-4o-class multimodal) instead of a blanket false, with the OCR capability resolver feeding the codex slug rather than the user's aiModel. A non-multimodal slug or an image-block rejection degrades to a clean extract failure / local-OCR text mode. --- src/lib/ai/__tests__/codex-client.test.ts | 62 +++++++++++++++++++ .../ai/__tests__/vision-capability.test.ts | 20 +++++- src/lib/ai/codex-client.ts | 41 +++++++++++- src/lib/ai/vision-capability.ts | 33 +++++++++- src/lib/labs/ocr-capability.ts | 5 ++ 5 files changed, 156 insertions(+), 5 deletions(-) diff --git a/src/lib/ai/__tests__/codex-client.test.ts b/src/lib/ai/__tests__/codex-client.test.ts index 7e6d77a2e..df0c1b06a 100644 --- a/src/lib/ai/__tests__/codex-client.test.ts +++ b/src/lib/ai/__tests__/codex-client.test.ts @@ -196,4 +196,66 @@ describe("CodexClient", () => { ).rejects.toThrow("Codex request failed (500)"); expect(onRefresh).not.toHaveBeenCalled(); }); + + it("emits input_image content blocks when images are present", async () => { + const mockFetch = vi + .fn() + .mockResolvedValue( + sseResponse([messageDoneEvent('{"summary":"ok"}'), completedEvent(50)]), + ); + vi.stubGlobal("fetch", mockFetch); + + const client = new CodexClient({ + accessToken: "test-token", + accountId: "acct-test", + onTokenRefresh: vi + .fn() + .mockResolvedValue({ accessToken: "x", accountId: "acct-test" }), + }); + + await client.generateCompletion({ + systemPrompt: "Transcribe.", + userPrompt: "Read this report.", + images: [{ mediaType: "image/png", dataBase64: "QUJD" }], + }); + + const body = JSON.parse(mockFetch.mock.calls[0][1].body); + const content = body.input[0].content; + expect(content[0]).toEqual({ + type: "input_text", + text: "Read this report.", + }); + expect(content[1]).toEqual({ + type: "input_image", + image_url: "data:image/png;base64,QUJD", + detail: "high", + }); + }); + + it("keeps the text-only content shape when no images are present", async () => { + const mockFetch = vi + .fn() + .mockResolvedValue( + sseResponse([messageDoneEvent('{"summary":"ok"}'), completedEvent(50)]), + ); + vi.stubGlobal("fetch", mockFetch); + + const client = new CodexClient({ + accessToken: "test-token", + accountId: "acct-test", + onTokenRefresh: vi + .fn() + .mockResolvedValue({ accessToken: "x", accountId: "acct-test" }), + }); + + await client.generateCompletion({ + systemPrompt: "Answer.", + userPrompt: "Plain text only.", + }); + + const body = JSON.parse(mockFetch.mock.calls[0][1].body); + expect(body.input[0].content).toEqual([ + { type: "input_text", text: "Plain text only." }, + ]); + }); }); diff --git a/src/lib/ai/__tests__/vision-capability.test.ts b/src/lib/ai/__tests__/vision-capability.test.ts index 362a27d8e..7edebe2fb 100644 --- a/src/lib/ai/__tests__/vision-capability.test.ts +++ b/src/lib/ai/__tests__/vision-capability.test.ts @@ -54,8 +54,24 @@ describe("supportsVisionForConfig", () => { expect(supportsVisionForConfig("local", null)).toBe(true); }); - it("never reports vision for codex or none", () => { - expect(supportsVisionForConfig("codex", "gpt-4o")).toBe(false); + it("reports vision for codex only on multimodal slugs", () => { + for (const model of [ + "gpt-5.5", + "gpt-5.4", + "gpt-5.4-mini", + "gpt-4o", + "gpt-4.1", + "o1", + ]) { + expect(supportsVisionForConfig("codex", model)).toBe(true); + } + // Non-multimodal / unknown slugs and a missing slug → false. + expect(supportsVisionForConfig("codex", "gpt-4")).toBe(false); + expect(supportsVisionForConfig("codex", "gpt-3.5-turbo")).toBe(false); + expect(supportsVisionForConfig("codex", null)).toBe(false); + }); + + it("never reports vision for none", () => { expect(supportsVisionForConfig("none", "gpt-4o")).toBe(false); }); }); diff --git a/src/lib/ai/codex-client.ts b/src/lib/ai/codex-client.ts index 4e08dc647..4ee0e7818 100644 --- a/src/lib/ai/codex-client.ts +++ b/src/lib/ai/codex-client.ts @@ -52,6 +52,20 @@ const DEFAULT_SLUG_FALLBACK_CHAIN = [ "gpt-5.2", // legacy floor ] as const; +/** + * The slug the codex vision gate reasons about *before* a request runs: the + * cached working slug if one is live, else the first chain entry. Lets the + * Lab-OCR capability resolver decide whether codex can read images without + * standing up a client or making a network call. Best-effort — the actual + * routed slug may differ once the chain walk runs, which the runtime probe + + * graceful fallback (text/local OCR) covers. + */ +export function resolveCodexVisionSlug(): string | null { + const cached = getCachedCodexSlug(); + if (cached) return cached; + return loadFallbackChain()[0] ?? null; +} + function loadFallbackChain(): string[] { const envChain = process.env.CODEX_MODEL_FALLBACK_CHAIN?.trim(); const fromEnv = envChain @@ -291,6 +305,31 @@ export class CodexClient implements AIProvider { ): Promise { const sessionId = randomUUID(); const threadId = randomUUID(); + + // v1.18.11 — fold vision inputs (Lab-OCR) into the user turn when present. + // The Codex/ChatGPT-OAuth backend accepts the `input_image` content block + // on this same `codex/responses` endpoint (docs/codex-protocol-spec.md + // §2b: `image_url` accepts a `data:;base64,...` URL, `detail` + // defaults to "high"). Image INPUT consumes the ChatGPT plan allocation — + // no API key is required (an API key only matters for image GENERATION). + // When no images are present the content is byte-identical to the prior + // text-only wire, so every text caller (Coach, insights, status cards) is + // untouched. The image is UNTRUSTED data to transcribe, framed as such by + // the system prompt. Documents (PDF) are NOT folded in — `input_image` is + // image-only; the OCR route gates PDFs to Anthropic. + const images = params.images ?? []; + const content: Array< + | { type: "input_text"; text: string } + | { type: "input_image"; image_url: string; detail: "high" } + > = [{ type: "input_text", text: params.userPrompt }]; + for (const img of images) { + content.push({ + type: "input_image", + image_url: `data:${img.mediaType};base64,${img.dataBase64}`, + detail: "high", + }); + } + return safeFetch( CODEX_ENDPOINT, { @@ -314,7 +353,7 @@ export class CodexClient implements AIProvider { { type: "message", role: "user", - content: [{ type: "input_text", text: params.userPrompt }], + content, }, ], tools: [], diff --git a/src/lib/ai/vision-capability.ts b/src/lib/ai/vision-capability.ts index 360b87160..10f0950fd 100644 --- a/src/lib/ai/vision-capability.ts +++ b/src/lib/ai/vision-capability.ts @@ -56,6 +56,31 @@ function openaiSupportsVision(model: string): boolean { ); } +/** + * Codex (ChatGPT-OAuth) vision models. Image INPUT is reachable over the + * `codex/responses` endpoint (docs/codex-protocol-spec.md §2b) on a multimodal + * slug — no API key needed. The accepted GPT-5.x line is text+vision; the older + * `-codex` specialist slugs and the legacy `gpt-4`-class non-multimodal pins + * are text-only. The slug we test is the codex client's resolved working slug + * (cached or chain head), so a slug rotation onto a non-vision model degrades + * to false rather than 400-ing the user at the image block. + * + * `gpt-4o` / `o*` are folded in for completeness (a future codex slug could + * land there), reusing the same multimodal families the OpenAI gate accepts. + */ +function codexSupportsVision(model: string): boolean { + const m = model.toLowerCase(); + return ( + m.startsWith("gpt-5") || // GPT-5.x line is multimodal (text + vision) + m.startsWith("gpt-4o") || + m.startsWith("gpt-4.1") || + m.startsWith("gpt-4-turbo") || + m.startsWith("o1") || + m.startsWith("o3") || + m.startsWith("o4") + ); +} + /** * Logical provider tags the OCR path reasons about. Mirrors the * `ProviderChainType` vocabulary plus the legacy `admin-key` single-provider @@ -79,8 +104,11 @@ export type VisionProviderType = * configuring a self-hosted model and we cannot sniff * its capabilities; a non-vision local model surfaces * a clear extract failure rather than a silent gate. - * - codex / none → false (no vision over the Codex wire; nothing - * configured). + * - codex → vision when the resolved codex slug is a + * multimodal model (GPT-5.x / gpt-4o-class). Image + * input rides the ChatGPT plan — no API key. A + * non-multimodal slug (or unknown) → false. + * - none → false (nothing configured). */ export function supportsVisionForConfig( providerType: VisionProviderType, @@ -96,6 +124,7 @@ export function supportsVisionForConfig( case "local": return true; case "codex": + return model ? codexSupportsVision(model) : false; case "none": default: return false; diff --git a/src/lib/labs/ocr-capability.ts b/src/lib/labs/ocr-capability.ts index fd760dd9c..400fc0187 100644 --- a/src/lib/labs/ocr-capability.ts +++ b/src/lib/labs/ocr-capability.ts @@ -11,6 +11,7 @@ */ import type { ProviderChainResolved } from "@/lib/ai/provider-runner"; import { resolveProvider, resolveProviderChain } from "@/lib/ai/provider"; +import { resolveCodexVisionSlug } from "@/lib/ai/codex-client"; import { prisma } from "@/lib/db"; import { supportsPdfForProvider, @@ -31,6 +32,10 @@ function modelForEntry( providerType: ProviderChainResolved["providerType"], ctx: ModelContext, ): string | null { + // Codex resolves its working slug at request time from the OAuth slug + // fallback chain — NOT from the user's `aiModel`. Use the cached/chain-head + // slug so the codex vision gate tests the model that actually runs. + if (providerType === "codex") return resolveCodexVisionSlug(); return providerType === "admin-openai" ? ctx.adminModel : ctx.userModel; }