feat(labs): unlock native Codex vision for lab OCR

Image input rides the ChatGPT-OAuth codex/responses endpoint via the
input_image content block (protocol spec 2b) — no vision API key. The
codex client folds params.images into input_image blocks alongside the
input_text; text-only calls keep the prior byte-identical wire. The
vision gate now resolves codex capability against the working slug
(GPT-5.x / gpt-4o-class multimodal) instead of a blanket false, with the
OCR capability resolver feeding the codex slug rather than the user's
aiModel. A non-multimodal slug or an image-block rejection degrades to a
clean extract failure / local-OCR text mode.
This commit is contained in:
Marc-André Bombeck committed 2026-06-20 18:59:37 +02:00
1 parent 4aebaaf399
commit 178d4d7610
5 files changed
+156 -5

No files matched your search

+62
View File
@@ -196,4 +196,66 @@ describe("CodexClient", () => {
).rejects.toThrow("Codex request failed (500)");
expect(onRefresh).not.toHaveBeenCalled();
});
it("emits input_image content blocks when images are present", async () => {
const mockFetch = vi
.fn()
.mockResolvedValue(
sseResponse([messageDoneEvent('{"summary":"ok"}'), completedEvent(50)]),
);
vi.stubGlobal("fetch", mockFetch);
const client = new CodexClient({
accessToken: "test-token",
accountId: "acct-test",
onTokenRefresh: vi
.fn()
.mockResolvedValue({ accessToken: "x", accountId: "acct-test" }),
});
await client.generateCompletion({
systemPrompt: "Transcribe.",
userPrompt: "Read this report.",
images: [{ mediaType: "image/png", dataBase64: "QUJD" }],
});
const body = JSON.parse(mockFetch.mock.calls[0][1].body);
const content = body.input[0].content;
expect(content[0]).toEqual({
type: "input_text",
text: "Read this report.",
});
expect(content[1]).toEqual({
type: "input_image",
image_url: "data:image/png;base64,QUJD",
detail: "high",
});
});
it("keeps the text-only content shape when no images are present", async () => {
const mockFetch = vi
.fn()
.mockResolvedValue(
sseResponse([messageDoneEvent('{"summary":"ok"}'), completedEvent(50)]),
);
vi.stubGlobal("fetch", mockFetch);
const client = new CodexClient({
accessToken: "test-token",
accountId: "acct-test",
onTokenRefresh: vi
.fn()
.mockResolvedValue({ accessToken: "x", accountId: "acct-test" }),
});
await client.generateCompletion({
systemPrompt: "Answer.",
userPrompt: "Plain text only.",
});
const body = JSON.parse(mockFetch.mock.calls[0][1].body);
expect(body.input[0].content).toEqual([
{ type: "input_text", text: "Plain text only." },
]);
});
});
+18 -2
View File
@@ -54,8 +54,24 @@ describe("supportsVisionForConfig", () => {
expect(supportsVisionForConfig("local", null)).toBe(true);
});
it("never reports vision for codex or none", () => {
expect(supportsVisionForConfig("codex", "gpt-4o")).toBe(false);
it("reports vision for codex only on multimodal slugs", () => {
for (const model of [
"gpt-5.5",
"gpt-5.4",
"gpt-5.4-mini",
"gpt-4o",
"gpt-4.1",
"o1",
]) {
expect(supportsVisionForConfig("codex", model)).toBe(true);
}
// Non-multimodal / unknown slugs and a missing slug → false.
expect(supportsVisionForConfig("codex", "gpt-4")).toBe(false);
expect(supportsVisionForConfig("codex", "gpt-3.5-turbo")).toBe(false);
expect(supportsVisionForConfig("codex", null)).toBe(false);
});
it("never reports vision for none", () => {
expect(supportsVisionForConfig("none", "gpt-4o")).toBe(false);
});
});