From 7f71f0e7a6ef7bd3b94ed55e4e64cd3fa8e66bf1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Sat, 20 Jun 2026 12:24:38 +0200 Subject: [PATCH] feat(ai): add optional vision inputs and capability resolution Extend CompletionParams with optional images[] and documents[] so the provider clients can carry an uploaded lab-report photo or PDF. Anthropic folds image and native document blocks into the user turn; OpenAI and the local OpenAI-compatible client fold image_url blocks. Text-only callers are untouched. Add supportsVisionForConfig / supportsPdfForProvider and an ocrExtract budget for the Lab-OCR ingestion path. --- src/lib/ai/__tests__/anthropic-client.test.ts | 66 +++++++++++ .../ai/__tests__/vision-capability.test.ts | 68 +++++++++++ src/lib/ai/ai-budgets.ts | 12 ++ src/lib/ai/anthropic-client.ts | 70 ++++++++++- src/lib/ai/local-client.ts | 21 +++- src/lib/ai/openai-client.ts | 22 +++- src/lib/ai/types.ts | 39 +++++++ src/lib/ai/vision-capability.ts | 109 ++++++++++++++++++ 8 files changed, 403 insertions(+), 4 deletions(-) create mode 100644 src/lib/ai/__tests__/vision-capability.test.ts create mode 100644 src/lib/ai/vision-capability.ts diff --git a/src/lib/ai/__tests__/anthropic-client.test.ts b/src/lib/ai/__tests__/anthropic-client.test.ts index 3db2580da..da32e8c77 100644 --- a/src/lib/ai/__tests__/anthropic-client.test.ts +++ b/src/lib/ai/__tests__/anthropic-client.test.ts @@ -179,6 +179,72 @@ describe("AnthropicClient", () => { expect(err.bodyExcerpt).not.toContain("supersecretkey"); }); + it("folds image + document blocks into the user turn for vision (Lab-OCR)", async () => { + const mockFetch = vi.fn().mockResolvedValue({ + ok: true, + json: () => + Promise.resolve({ + content: [{ type: "text", text: '"rows":[]}' }], + usage: { input_tokens: 100, output_tokens: 10 }, + }), + }); + vi.stubGlobal("fetch", mockFetch); + + const client = new AnthropicClient({ + apiKey: "sk-ant-x", + model: "claude-sonnet-4-6", + }); + + await client.generateCompletion({ + systemPrompt: "Transcribe this report.", + userPrompt: "Extract the readings.", + responseFormat: "json", + images: [{ mediaType: "image/jpeg", dataBase64: "aW1n" }], + documents: [{ mediaType: "application/pdf", dataBase64: "cGRm" }], + }); + + const body = JSON.parse(mockFetch.mock.calls[0][1].body); + // First message is the user turn; its content is now a typed array. + const content = body.messages[0].content; + expect(Array.isArray(content)).toBe(true); + const types = content.map((b: { type: string }) => b.type); + expect(types).toContain("image"); + expect(types).toContain("document"); + // The instruction text comes last, after the data blocks. + expect(content[content.length - 1].type).toBe("text"); + const image = content.find((b: { type: string }) => b.type === "image"); + expect(image.source).toEqual({ + type: "base64", + media_type: "image/jpeg", + data: "aW1n", + }); + const doc = content.find((b: { type: string }) => b.type === "document"); + expect(doc.source.media_type).toBe("application/pdf"); + // The assistant `{`-prefill still rides after the array content. + expect(body.messages[1]).toEqual({ role: "assistant", content: "{" }); + }); + + it("keeps a bare-string user content when no vision input is present", async () => { + const mockFetch = vi.fn().mockResolvedValue({ + ok: true, + json: () => + Promise.resolve({ + content: [{ type: "text", text: "{}" }], + usage: { input_tokens: 1, output_tokens: 1 }, + }), + }); + vi.stubGlobal("fetch", mockFetch); + + const client = new AnthropicClient({ + apiKey: "sk-ant-x", + model: "claude-sonnet-4-6", + }); + + await client.generateCompletion({ systemPrompt: "s", userPrompt: "u" }); + const body = JSON.parse(mockFetch.mock.calls[0][1].body); + expect(typeof body.messages[0].content).toBe("string"); + }); + it("throws when content is empty", async () => { vi.stubGlobal( "fetch", diff --git a/src/lib/ai/__tests__/vision-capability.test.ts b/src/lib/ai/__tests__/vision-capability.test.ts new file mode 100644 index 000000000..7742ae293 --- /dev/null +++ b/src/lib/ai/__tests__/vision-capability.test.ts @@ -0,0 +1,68 @@ +import { describe, it, expect } from "vitest"; + +import { + supportsVisionForConfig, + supportsPdfForProvider, +} from "../vision-capability"; + +describe("supportsVisionForConfig", () => { + it("accepts Claude 3.5+/4 family models for anthropic", () => { + for (const model of [ + "claude-3-5-sonnet-latest", + "claude-3-7-sonnet", + "claude-sonnet-4-6", + "claude-opus-4-1", + "claude-haiku-4", + ]) { + expect(supportsVisionForConfig("anthropic", model)).toBe(true); + } + }); + + it("rejects legacy / text-only anthropic models", () => { + expect(supportsVisionForConfig("anthropic", "claude-2.1")).toBe(false); + expect(supportsVisionForConfig("anthropic", "claude-instant-1")).toBe( + false, + ); + expect(supportsVisionForConfig("anthropic", null)).toBe(false); + }); + + it("accepts gpt-4o/4.1/turbo/o-series for openai + admin variants", () => { + for (const providerType of [ + "openai", + "admin-openai", + "admin-key", + ] as const) { + expect(supportsVisionForConfig(providerType, "gpt-4o")).toBe(true); + expect(supportsVisionForConfig(providerType, "gpt-4o-mini")).toBe(true); + expect(supportsVisionForConfig(providerType, "gpt-4.1")).toBe(true); + expect(supportsVisionForConfig(providerType, "gpt-4-turbo")).toBe(true); + expect(supportsVisionForConfig(providerType, "o1")).toBe(true); + } + }); + + it("rejects non-vision openai models", () => { + expect(supportsVisionForConfig("openai", "gpt-3.5-turbo")).toBe(false); + expect(supportsVisionForConfig("openai", "gpt-4")).toBe(false); + expect(supportsVisionForConfig("openai", null)).toBe(false); + }); + + it("trusts local (operator opted in) regardless of model string", () => { + expect(supportsVisionForConfig("local", "llava")).toBe(true); + expect(supportsVisionForConfig("local", null)).toBe(true); + }); + + it("never reports vision for codex or none", () => { + expect(supportsVisionForConfig("codex", "gpt-4o")).toBe(false); + expect(supportsVisionForConfig("none", "gpt-4o")).toBe(false); + }); +}); + +describe("supportsPdfForProvider", () => { + it("only anthropic supports native PDF", () => { + expect(supportsPdfForProvider("anthropic")).toBe(true); + expect(supportsPdfForProvider("openai")).toBe(false); + expect(supportsPdfForProvider("admin-openai")).toBe(false); + expect(supportsPdfForProvider("local")).toBe(false); + expect(supportsPdfForProvider("codex")).toBe(false); + }); +}); diff --git a/src/lib/ai/ai-budgets.ts b/src/lib/ai/ai-budgets.ts index df64ee566..e0e67d14e 100644 --- a/src/lib/ai/ai-budgets.ts +++ b/src/lib/ai/ai-budgets.ts @@ -97,4 +97,16 @@ export const AI_BUDGETS = { * questions. Temperature only; the caller pins its own token const. */ selfContext: { temperature: 0.4 }, + + /** + * v1.18.9 — Lab-OCR structured extraction. ONE vision call transcribes a + * photo / PDF of a paper lab report into a JSON array of analyte rows + * (analyte + value/valueText + unit + reference range + date + per-field + * confidence). A full panel is a dozen-plus rows of compact JSON; 4000 + * tokens covers a dense report plus the envelope. Temperature 0 — the task + * is faithful transcription, not generation, so the lowest setting minimises + * invented values. Vision input tokens are expensive, so the route also + * gates with a tight 6/hour rate bucket + reserveBudget + upload-size cap. + */ + ocrExtract: { temperature: 0, maxTokens: 4000 }, } as const satisfies Record; diff --git a/src/lib/ai/anthropic-client.ts b/src/lib/ai/anthropic-client.ts index 87ac4d5af..9f99bdb81 100644 --- a/src/lib/ai/anthropic-client.ts +++ b/src/lib/ai/anthropic-client.ts @@ -1,6 +1,63 @@ import { safeFetch } from "@/lib/safe-fetch"; import type { AIProvider, CompletionParams, CompletionResult } from "./types"; +/** + * v1.18.9 — Anthropic Messages API content blocks. The text-only path sends a + * bare string; the vision path (Lab-OCR) sends an array of typed blocks. The + * uploaded image / PDF is framed strictly as DATA to transcribe by the system + * prompt — never as instructions (prompt-injection backstop is the human + * review screen downstream). + */ +type AnthropicContentBlock = + | { type: "text"; text: string } + | { + type: "image"; + source: { type: "base64"; media_type: string; data: string }; + } + | { + type: "document"; + source: { type: "base64"; media_type: "application/pdf"; data: string }; + }; + +/** + * Build the user-turn content for a vision request: the image / document + * blocks first (so the model reads the report before the instruction), then + * the JSON-wrapped instruction text. Returns null when there is nothing to + * attach, so the caller keeps the bare-string text path unchanged. + */ +function buildVisionContent( + params: CompletionParams, + text: string, +): AnthropicContentBlock[] | null { + const images = params.images ?? []; + const documents = params.documents ?? []; + if (images.length === 0 && documents.length === 0) return null; + + const blocks: AnthropicContentBlock[] = []; + for (const img of images) { + blocks.push({ + type: "image", + source: { + type: "base64", + media_type: img.mediaType, + data: img.dataBase64, + }, + }); + } + for (const doc of documents) { + blocks.push({ + type: "document", + source: { + type: "base64", + media_type: "application/pdf", + data: doc.dataBase64, + }, + }); + } + blocks.push({ type: "text", text }); + return blocks; +} + interface AnthropicClientConfig { apiKey: string; model: string; @@ -36,6 +93,15 @@ export class AnthropicClient implements AIProvider { ); const url = `${baseUrl}/messages`; + // v1.18.9 — fold vision inputs (Lab-OCR) into the user turn when present. + // The text-only path keeps a bare-string content; the vision path sends a + // typed content array (image / document blocks + the instruction text). + // The `{`-prefill JSON trick works after either shape. + const wrappedText = wrapForJson(params.userPrompt); + const visionContent = buildVisionContent(params, wrappedText); + const userContent: AnthropicContentBlock[] | string = + visionContent ?? wrappedText; + const res = await safeFetch( url, { @@ -64,10 +130,10 @@ export class AnthropicClient implements AIProvider { messages: params.responseFormat === "json" ? [ - { role: "user", content: wrapForJson(params.userPrompt) }, + { role: "user", content: userContent }, { role: "assistant", content: "{" }, ] - : [{ role: "user", content: wrapForJson(params.userPrompt) }], + : [{ role: "user", content: userContent }], }), }, // 60 s ceiling — see openai-client.ts for the rationale. diff --git a/src/lib/ai/local-client.ts b/src/lib/ai/local-client.ts index 19abec405..2843a050f 100644 --- a/src/lib/ai/local-client.ts +++ b/src/lib/ai/local-client.ts @@ -40,6 +40,25 @@ export class LocalOpenAICompatibleClient implements AIProvider { const userPrompt = `Return strict JSON only, no markdown, no commentary.\n\n${params.userPrompt}`; + // v1.18.9 — fold vision inputs (Lab-OCR) into the user turn when present, + // using the OpenAI-compatible `image_url` content array that llava / + // llama-vision models accept over the same endpoint. PDFs are not handled + // here (the OCR route gates PDF to Anthropic); only `params.images` is + // consumed. The image is framed as untrusted DATA by the system prompt. + const images = params.images ?? []; + const userContent = + images.length > 0 + ? [ + { type: "text" as const, text: userPrompt }, + ...images.map((img) => ({ + type: "image_url" as const, + image_url: { + url: `data:${img.mediaType};base64,${img.dataBase64}`, + }, + })), + ] + : userPrompt; + // safeFetch defaults: no redirect-follow (the local endpoint is the // most exploitable on this surface — a user-controlled baseUrl that // 302s to 169.254.169.254 would otherwise leak the bearer on the @@ -64,7 +83,7 @@ export class LocalOpenAICompatibleClient implements AIProvider { model: this.config.model, messages: [ { role: "system", content: params.systemPrompt }, - { role: "user", content: userPrompt }, + { role: "user", content: userContent }, ], temperature: params.temperature ?? 0.3, max_tokens: params.maxTokens ?? 1000, diff --git a/src/lib/ai/openai-client.ts b/src/lib/ai/openai-client.ts index 7260e6320..bd1f67880 100644 --- a/src/lib/ai/openai-client.ts +++ b/src/lib/ai/openai-client.ts @@ -31,6 +31,26 @@ export class OpenAIClient implements AIProvider { ): Promise { const url = `${this.config.baseUrl.replace(/\/$/, "")}/chat/completions`; + // v1.18.9 — fold vision inputs (Lab-OCR) into the user turn when present. + // The text-only path keeps the bare-string content; the vision path sends + // the multimodal `image_url` content array `gpt-4o`-class models accept + // (a data-URL per image). PDFs are NOT handled here (the OCR route gates + // PDF uploads to Anthropic); only `params.images` is consumed. The image + // is framed as untrusted DATA by the system prompt. + const images = params.images ?? []; + const userContent = + images.length > 0 + ? [ + { type: "text" as const, text: params.userPrompt }, + ...images.map((img) => ({ + type: "image_url" as const, + image_url: { + url: `data:${img.mediaType};base64,${img.dataBase64}`, + }, + })), + ] + : params.userPrompt; + const res = await safeFetch( url, { @@ -43,7 +63,7 @@ export class OpenAIClient implements AIProvider { model: this.config.model, messages: [ { role: "system", content: params.systemPrompt }, - { role: "user", content: params.userPrompt }, + { role: "user", content: userContent }, ], temperature: params.temperature ?? 0.3, max_tokens: params.maxTokens ?? 1000, diff --git a/src/lib/ai/types.ts b/src/lib/ai/types.ts index d5844da2c..936ad4a11 100644 --- a/src/lib/ai/types.ts +++ b/src/lib/ai/types.ts @@ -114,11 +114,50 @@ export interface AIProvider { generateCompletion(params: CompletionParams): Promise; } +/** + * v1.18.9 — a single vision input folded into the provider request. Used by + * the Lab-OCR extraction path: a photo of a paper report, or a PDF page. + * `dataBase64` is the raw file bytes, base64-encoded (no `data:` prefix — + * each client wraps it in the wire shape it needs). The blob is NEVER threaded + * into an `annotate()` meta or any log field (it is health data). + */ +export interface CompletionImage { + /** One of the sniffed image MIME types. */ + mediaType: "image/jpeg" | "image/png" | "image/webp"; + /** Raw file bytes, base64-encoded (no data-URL prefix). */ + dataBase64: string; +} + +/** + * v1.18.9 — a PDF document folded into the provider request. Only Anthropic + * accepts a native `document` block over the Messages API we use; the other + * clients ignore it (the OCR route gates PDFs to Anthropic providers). + */ +export interface CompletionDocument { + mediaType: "application/pdf"; + /** Raw PDF bytes, base64-encoded (no data-URL prefix). */ + dataBase64: string; +} + export interface CompletionParams { systemPrompt: string; userPrompt: string; temperature?: number; maxTokens?: number; + /** + * v1.18.9 — optional vision inputs (Lab-OCR). Additive: every text-only + * caller (Coach, status cards, insights, narratives) leaves these unset, so + * the existing wire shape and streaming path are untouched. A client that + * cannot read images/documents ignores the field. Treat the contents as + * UNTRUSTED data to transcribe — never as instructions. + */ + images?: CompletionImage[]; + /** + * v1.18.9 — optional PDF document input (Lab-OCR). Anthropic-only; other + * clients ignore it. Mutually informative with `images` — the OCR route + * sends one or the other depending on the upload type and provider. + */ + documents?: CompletionDocument[]; /** * Optional deterministic seed. Threaded onto the OpenAI and local * (Ollama / OpenAI-compatible) request bodies for reproducible output on diff --git a/src/lib/ai/vision-capability.ts b/src/lib/ai/vision-capability.ts new file mode 100644 index 000000000..2ab906afb --- /dev/null +++ b/src/lib/ai/vision-capability.ts @@ -0,0 +1,109 @@ +/** + * v1.18.9 — vision-capability resolution for the Lab-OCR ingestion path. + * + * The provider abstraction is text-only by default; only some providers + + * models can read an uploaded photo / PDF. This module is the single source of + * truth for "can the user's configured provider read images?", so the upload + * route and the capability probe endpoint agree. + * + * Capability is a function of (providerType, model), evaluated against the + * documented vision allowlists below. It is PURE and presence-only — it never + * decrypts a key, constructs a client, or probes liveness. A `true` here can + * still resolve to a dead provider at call time (revoked key, unreachable + * local host); the route surfaces that as an extract failure. + */ + +/** + * Anthropic vision models. The Messages API `image` + `document` blocks are + * accepted by the Claude 3.5+ / 3.7 / 4 families. An older `claude-2*` / + * `claude-instant*` pin is text-only. + */ +function anthropicSupportsVision(model: string): boolean { + const m = model.toLowerCase(); + return ( + m.startsWith("claude-3-5") || + m.startsWith("claude-3.5") || + m.startsWith("claude-3-7") || + m.startsWith("claude-3.7") || + m.startsWith("claude-sonnet-4") || + m.startsWith("claude-opus-4") || + m.startsWith("claude-haiku-4") + ); +} + +/** + * OpenAI vision models (also covers the operator's admin key + Codex when it + * runs an OpenAI model — though Codex is excluded by `providerType`). `gpt-4o`, + * `gpt-4.1`, `gpt-4-turbo`, and the `o*` reasoning models read image content; + * `gpt-3.5*` and the legacy `gpt-4` (non-turbo) 400 on an image block. + */ +function openaiSupportsVision(model: string): boolean { + const m = model.toLowerCase(); + return ( + m.startsWith("gpt-4o") || + m.startsWith("gpt-4.1") || + m.startsWith("gpt-4-turbo") || + m.startsWith("o1") || + m.startsWith("o3") || + m.startsWith("o4") + ); +} + +/** + * Logical provider tags the OCR path reasons about. Mirrors the + * `ProviderChainType` vocabulary plus the legacy `admin-key` single-provider + * tag `resolveProvider`'s admin fallback returns. + */ +export type VisionProviderType = + | "anthropic" + | "openai" + | "admin-openai" + | "admin-key" + | "local" + | "codex" + | "none"; + +/** + * Pure capability check over a (providerType, model) pair. + * + * - anthropic → vision when the model is a 3.5+/4 family pin. + * - openai / admin-* → vision when the model is a gpt-4o/4.1/turbo/o* pin. + * - local → trust-by-default true: the operator opted in by + * configuring a self-hosted model and we cannot sniff + * its capabilities; a non-vision local model surfaces + * a clear extract failure rather than a silent gate. + * - codex / none → false (no vision over the Codex wire; nothing + * configured). + */ +export function supportsVisionForConfig( + providerType: VisionProviderType, + model: string | null, +): boolean { + switch (providerType) { + case "anthropic": + return model ? anthropicSupportsVision(model) : false; + case "openai": + case "admin-openai": + case "admin-key": + return model ? openaiSupportsVision(model) : false; + case "local": + return true; + case "codex": + case "none": + default: + return false; + } +} + +/** + * Whether PDFs are supported for a given provider. Only Anthropic accepts a + * native `document` block over the Messages API; OpenAI / local read images + * only (a server-side rasterizer is a deferred follow-up). The route uses this + * to reject a PDF upload with a clear "use a photo / configure Claude" message + * on a non-Anthropic provider. + */ +export function supportsPdfForProvider( + providerType: VisionProviderType, +): boolean { + return providerType === "anthropic"; +}