mirror of
https://github.com/MBombeck/HealthLog.git
synced 2026-11-10 13:22:14 +00:00
feat(labs): unlock native Codex vision for lab OCR
Image input rides the ChatGPT-OAuth codex/responses endpoint via the input_image content block (protocol spec 2b) — no vision API key. The codex client folds params.images into input_image blocks alongside the input_text; text-only calls keep the prior byte-identical wire. The vision gate now resolves codex capability against the working slug (GPT-5.x / gpt-4o-class multimodal) instead of a blanket false, with the OCR capability resolver feeding the codex slug rather than the user's aiModel. A non-multimodal slug or an image-block rejection degrades to a clean extract failure / local-OCR text mode.
This commit is contained in:
1 parent
4aebaaf399
commit
178d4d7610
5 files changed
+156
-5
No files matched your search
@@ -196,4 +196,66 @@ describe("CodexClient", () => {
|
||||
).rejects.toThrow("Codex request failed (500)");
|
||||
expect(onRefresh).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("emits input_image content blocks when images are present", async () => {
|
||||
const mockFetch = vi
|
||||
.fn()
|
||||
.mockResolvedValue(
|
||||
sseResponse([messageDoneEvent('{"summary":"ok"}'), completedEvent(50)]),
|
||||
);
|
||||
vi.stubGlobal("fetch", mockFetch);
|
||||
|
||||
const client = new CodexClient({
|
||||
accessToken: "test-token",
|
||||
accountId: "acct-test",
|
||||
onTokenRefresh: vi
|
||||
.fn()
|
||||
.mockResolvedValue({ accessToken: "x", accountId: "acct-test" }),
|
||||
});
|
||||
|
||||
await client.generateCompletion({
|
||||
systemPrompt: "Transcribe.",
|
||||
userPrompt: "Read this report.",
|
||||
images: [{ mediaType: "image/png", dataBase64: "QUJD" }],
|
||||
});
|
||||
|
||||
const body = JSON.parse(mockFetch.mock.calls[0][1].body);
|
||||
const content = body.input[0].content;
|
||||
expect(content[0]).toEqual({
|
||||
type: "input_text",
|
||||
text: "Read this report.",
|
||||
});
|
||||
expect(content[1]).toEqual({
|
||||
type: "input_image",
|
||||
image_url: "data:image/png;base64,QUJD",
|
||||
detail: "high",
|
||||
});
|
||||
});
|
||||
|
||||
it("keeps the text-only content shape when no images are present", async () => {
|
||||
const mockFetch = vi
|
||||
.fn()
|
||||
.mockResolvedValue(
|
||||
sseResponse([messageDoneEvent('{"summary":"ok"}'), completedEvent(50)]),
|
||||
);
|
||||
vi.stubGlobal("fetch", mockFetch);
|
||||
|
||||
const client = new CodexClient({
|
||||
accessToken: "test-token",
|
||||
accountId: "acct-test",
|
||||
onTokenRefresh: vi
|
||||
.fn()
|
||||
.mockResolvedValue({ accessToken: "x", accountId: "acct-test" }),
|
||||
});
|
||||
|
||||
await client.generateCompletion({
|
||||
systemPrompt: "Answer.",
|
||||
userPrompt: "Plain text only.",
|
||||
});
|
||||
|
||||
const body = JSON.parse(mockFetch.mock.calls[0][1].body);
|
||||
expect(body.input[0].content).toEqual([
|
||||
{ type: "input_text", text: "Plain text only." },
|
||||
]);
|
||||
});
|
||||
});
|
||||
@@ -54,8 +54,24 @@ describe("supportsVisionForConfig", () => {
|
||||
expect(supportsVisionForConfig("local", null)).toBe(true);
|
||||
});
|
||||
|
||||
it("never reports vision for codex or none", () => {
|
||||
expect(supportsVisionForConfig("codex", "gpt-4o")).toBe(false);
|
||||
it("reports vision for codex only on multimodal slugs", () => {
|
||||
for (const model of [
|
||||
"gpt-5.5",
|
||||
"gpt-5.4",
|
||||
"gpt-5.4-mini",
|
||||
"gpt-4o",
|
||||
"gpt-4.1",
|
||||
"o1",
|
||||
]) {
|
||||
expect(supportsVisionForConfig("codex", model)).toBe(true);
|
||||
}
|
||||
// Non-multimodal / unknown slugs and a missing slug → false.
|
||||
expect(supportsVisionForConfig("codex", "gpt-4")).toBe(false);
|
||||
expect(supportsVisionForConfig("codex", "gpt-3.5-turbo")).toBe(false);
|
||||
expect(supportsVisionForConfig("codex", null)).toBe(false);
|
||||
});
|
||||
|
||||
it("never reports vision for none", () => {
|
||||
expect(supportsVisionForConfig("none", "gpt-4o")).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
Reference in new issue
Block a user