feat(ai): add optional vision inputs and capability resolution

Extend CompletionParams with optional images[] and documents[] so the
provider clients can carry an uploaded lab-report photo or PDF. Anthropic
folds image and native document blocks into the user turn; OpenAI and the
local OpenAI-compatible client fold image_url blocks. Text-only callers are
untouched. Add supportsVisionForConfig / supportsPdfForProvider and an
ocrExtract budget for the Lab-OCR ingestion path.

src/lib/ai/__tests__/anthropic-client.test.ts

@@ -179,6 +179,72 @@ describe("AnthropicClient", () => {
     expect(err.bodyExcerpt).not.toContain("supersecretkey");
   });
 
+  it("folds image + document blocks into the user turn for vision (Lab-OCR)", async () => {
+    const mockFetch = vi.fn().mockResolvedValue({
+      ok: true,
+      json: () =>
+        Promise.resolve({
+          content: [{ type: "text", text: '"rows":[]}' }],
+          usage: { input_tokens: 100, output_tokens: 10 },
+        }),
+    });
+    vi.stubGlobal("fetch", mockFetch);
+
+    const client = new AnthropicClient({
+      apiKey: "sk-ant-x",
+      model: "claude-sonnet-4-6",
+    });
+
+    await client.generateCompletion({
+      systemPrompt: "Transcribe this report.",
+      userPrompt: "Extract the readings.",
+      responseFormat: "json",
+      images: [{ mediaType: "image/jpeg", dataBase64: "aW1n" }],
+      documents: [{ mediaType: "application/pdf", dataBase64: "cGRm" }],
+    });
+
+    const body = JSON.parse(mockFetch.mock.calls[0][1].body);
+    // First message is the user turn; its content is now a typed array.
+    const content = body.messages[0].content;
+    expect(Array.isArray(content)).toBe(true);
+    const types = content.map((b: { type: string }) => b.type);
+    expect(types).toContain("image");
+    expect(types).toContain("document");
+    // The instruction text comes last, after the data blocks.
+    expect(content[content.length - 1].type).toBe("text");
+    const image = content.find((b: { type: string }) => b.type === "image");
+    expect(image.source).toEqual({
+      type: "base64",
+      media_type: "image/jpeg",
+      data: "aW1n",
+    });
+    const doc = content.find((b: { type: string }) => b.type === "document");
+    expect(doc.source.media_type).toBe("application/pdf");
+    // The assistant `{`-prefill still rides after the array content.
+    expect(body.messages[1]).toEqual({ role: "assistant", content: "{" });
+  });
+
+  it("keeps a bare-string user content when no vision input is present", async () => {
+    const mockFetch = vi.fn().mockResolvedValue({
+      ok: true,
+      json: () =>
+        Promise.resolve({
+          content: [{ type: "text", text: "{}" }],
+          usage: { input_tokens: 1, output_tokens: 1 },
+        }),
+    });
+    vi.stubGlobal("fetch", mockFetch);
+
+    const client = new AnthropicClient({
+      apiKey: "sk-ant-x",
+      model: "claude-sonnet-4-6",
+    });
+
+    await client.generateCompletion({ systemPrompt: "s", userPrompt: "u" });
+    const body = JSON.parse(mockFetch.mock.calls[0][1].body);
+    expect(typeof body.messages[0].content).toBe("string");
+  });
+
   it("throws when content is empty", async () => {
     vi.stubGlobal(
       "fetch",

src/lib/ai/__tests__/vision-capability.test.ts

@@ -0,0 +1,68 @@
+import { describe, it, expect } from "vitest";
+
+import {
+  supportsVisionForConfig,
+  supportsPdfForProvider,
+} from "../vision-capability";
+
+describe("supportsVisionForConfig", () => {
+  it("accepts Claude 3.5+/4 family models for anthropic", () => {
+    for (const model of [
+      "claude-3-5-sonnet-latest",
+      "claude-3-7-sonnet",
+      "claude-sonnet-4-6",
+      "claude-opus-4-1",
+      "claude-haiku-4",
+    ]) {
+      expect(supportsVisionForConfig("anthropic", model)).toBe(true);
+    }
+  });
+
+  it("rejects legacy / text-only anthropic models", () => {
+    expect(supportsVisionForConfig("anthropic", "claude-2.1")).toBe(false);
+    expect(supportsVisionForConfig("anthropic", "claude-instant-1")).toBe(
+      false,
+    );
+    expect(supportsVisionForConfig("anthropic", null)).toBe(false);
+  });
+
+  it("accepts gpt-4o/4.1/turbo/o-series for openai + admin variants", () => {
+    for (const providerType of [
+      "openai",
+      "admin-openai",
+      "admin-key",
+    ] as const) {
+      expect(supportsVisionForConfig(providerType, "gpt-4o")).toBe(true);
+      expect(supportsVisionForConfig(providerType, "gpt-4o-mini")).toBe(true);
+      expect(supportsVisionForConfig(providerType, "gpt-4.1")).toBe(true);
+      expect(supportsVisionForConfig(providerType, "gpt-4-turbo")).toBe(true);
+      expect(supportsVisionForConfig(providerType, "o1")).toBe(true);
+    }
+  });
+
+  it("rejects non-vision openai models", () => {
+    expect(supportsVisionForConfig("openai", "gpt-3.5-turbo")).toBe(false);
+    expect(supportsVisionForConfig("openai", "gpt-4")).toBe(false);
+    expect(supportsVisionForConfig("openai", null)).toBe(false);
+  });
+
+  it("trusts local (operator opted in) regardless of model string", () => {
+    expect(supportsVisionForConfig("local", "llava")).toBe(true);
+    expect(supportsVisionForConfig("local", null)).toBe(true);
+  });
+
+  it("never reports vision for codex or none", () => {
+    expect(supportsVisionForConfig("codex", "gpt-4o")).toBe(false);
+    expect(supportsVisionForConfig("none", "gpt-4o")).toBe(false);
+  });
+});
+
+describe("supportsPdfForProvider", () => {
+  it("only anthropic supports native PDF", () => {
+    expect(supportsPdfForProvider("anthropic")).toBe(true);
+    expect(supportsPdfForProvider("openai")).toBe(false);
+    expect(supportsPdfForProvider("admin-openai")).toBe(false);
+    expect(supportsPdfForProvider("local")).toBe(false);
+    expect(supportsPdfForProvider("codex")).toBe(false);
+  });
+});

src/lib/ai/ai-budgets.ts

@@ -97,4 +97,16 @@ export const AI_BUDGETS = {
    * questions. Temperature only; the caller pins its own token const.
    */
   selfContext: { temperature: 0.4 },
+
+  /**
+   * v1.18.9 — Lab-OCR structured extraction. ONE vision call transcribes a
+   * photo / PDF of a paper lab report into a JSON array of analyte rows
+   * (analyte + value/valueText + unit + reference range + date + per-field
+   * confidence). A full panel is a dozen-plus rows of compact JSON; 4000
+   * tokens covers a dense report plus the envelope. Temperature 0 — the task
+   * is faithful transcription, not generation, so the lowest setting minimises
+   * invented values. Vision input tokens are expensive, so the route also
+   * gates with a tight 6/hour rate bucket + reserveBudget + upload-size cap.
+   */
+  ocrExtract: { temperature: 0, maxTokens: 4000 },
 } as const satisfies Record<string, AiBudget>;

src/lib/ai/anthropic-client.ts

@@ -1,6 +1,63 @@
 import { safeFetch } from "@/lib/safe-fetch";
 import type { AIProvider, CompletionParams, CompletionResult } from "./types";
 
+/**
+ * v1.18.9 — Anthropic Messages API content blocks. The text-only path sends a
+ * bare string; the vision path (Lab-OCR) sends an array of typed blocks. The
+ * uploaded image / PDF is framed strictly as DATA to transcribe by the system
+ * prompt — never as instructions (prompt-injection backstop is the human
+ * review screen downstream).
+ */
+type AnthropicContentBlock =
+  | { type: "text"; text: string }
+  | {
+      type: "image";
+      source: { type: "base64"; media_type: string; data: string };
+    }
+  | {
+      type: "document";
+      source: { type: "base64"; media_type: "application/pdf"; data: string };
+    };
+
+/**
+ * Build the user-turn content for a vision request: the image / document
+ * blocks first (so the model reads the report before the instruction), then
+ * the JSON-wrapped instruction text. Returns null when there is nothing to
+ * attach, so the caller keeps the bare-string text path unchanged.
+ */
+function buildVisionContent(
+  params: CompletionParams,
+  text: string,
+): AnthropicContentBlock[] | null {
+  const images = params.images ?? [];
+  const documents = params.documents ?? [];
+  if (images.length === 0 && documents.length === 0) return null;
+
+  const blocks: AnthropicContentBlock[] = [];
+  for (const img of images) {
+    blocks.push({
+      type: "image",
+      source: {
+        type: "base64",
+        media_type: img.mediaType,
+        data: img.dataBase64,
+      },
+    });
+  }
+  for (const doc of documents) {
+    blocks.push({
+      type: "document",
+      source: {
+        type: "base64",
+        media_type: "application/pdf",
+        data: doc.dataBase64,
+      },
+    });
+  }
+  blocks.push({ type: "text", text });
+  return blocks;
+}
+
 interface AnthropicClientConfig {
   apiKey: string;
   model: string;
@@ -36,6 +93,15 @@ export class AnthropicClient implements AIProvider {
     );
     const url = `${baseUrl}/messages`;
 
+    // v1.18.9 — fold vision inputs (Lab-OCR) into the user turn when present.
+    // The text-only path keeps a bare-string content; the vision path sends a
+    // typed content array (image / document blocks + the instruction text).
+    // The `{`-prefill JSON trick works after either shape.
+    const wrappedText = wrapForJson(params.userPrompt);
+    const visionContent = buildVisionContent(params, wrappedText);
+    const userContent: AnthropicContentBlock[] | string =
+      visionContent ?? wrappedText;
+
     const res = await safeFetch(
       url,
       {
@@ -64,10 +130,10 @@ export class AnthropicClient implements AIProvider {
           messages:
             params.responseFormat === "json"
               ? [
-                  { role: "user", content: wrapForJson(params.userPrompt) },
+                  { role: "user", content: userContent },
                   { role: "assistant", content: "{" },
                 ]
-              : [{ role: "user", content: wrapForJson(params.userPrompt) }],
+              : [{ role: "user", content: userContent }],
         }),
       },
       // 60 s ceiling — see openai-client.ts for the rationale.

src/lib/ai/local-client.ts

@@ -40,6 +40,25 @@ export class LocalOpenAICompatibleClient implements AIProvider {
 
     const userPrompt = `Return strict JSON only, no markdown, no commentary.\n\n${params.userPrompt}`;
 
+    // v1.18.9 — fold vision inputs (Lab-OCR) into the user turn when present,
+    // using the OpenAI-compatible `image_url` content array that llava /
+    // llama-vision models accept over the same endpoint. PDFs are not handled
+    // here (the OCR route gates PDF to Anthropic); only `params.images` is
+    // consumed. The image is framed as untrusted DATA by the system prompt.
+    const images = params.images ?? [];
+    const userContent =
+      images.length > 0
+        ? [
+            { type: "text" as const, text: userPrompt },
+            ...images.map((img) => ({
+              type: "image_url" as const,
+              image_url: {
+                url: `data:${img.mediaType};base64,${img.dataBase64}`,
+              },
+            })),
+          ]
+        : userPrompt;
+
     // safeFetch defaults: no redirect-follow (the local endpoint is the
     // most exploitable on this surface — a user-controlled baseUrl that
     // 302s to 169.254.169.254 would otherwise leak the bearer on the
@@ -64,7 +83,7 @@ export class LocalOpenAICompatibleClient implements AIProvider {
           model: this.config.model,
           messages: [
             { role: "system", content: params.systemPrompt },
-            { role: "user", content: userPrompt },
+            { role: "user", content: userContent },
           ],
           temperature: params.temperature ?? 0.3,
           max_tokens: params.maxTokens ?? 1000,

src/lib/ai/openai-client.ts

@@ -31,6 +31,26 @@ export class OpenAIClient implements AIProvider {
   ): Promise<CompletionResult> {
     const url = `${this.config.baseUrl.replace(/\/$/, "")}/chat/completions`;
 
+    // v1.18.9 — fold vision inputs (Lab-OCR) into the user turn when present.
+    // The text-only path keeps the bare-string content; the vision path sends
+    // the multimodal `image_url` content array `gpt-4o`-class models accept
+    // (a data-URL per image). PDFs are NOT handled here (the OCR route gates
+    // PDF uploads to Anthropic); only `params.images` is consumed. The image
+    // is framed as untrusted DATA by the system prompt.
+    const images = params.images ?? [];
+    const userContent =
+      images.length > 0
+        ? [
+            { type: "text" as const, text: params.userPrompt },
+            ...images.map((img) => ({
+              type: "image_url" as const,
+              image_url: {
+                url: `data:${img.mediaType};base64,${img.dataBase64}`,
+              },
+            })),
+          ]
+        : params.userPrompt;
+
     const res = await safeFetch(
       url,
       {
@@ -43,7 +63,7 @@ export class OpenAIClient implements AIProvider {
           model: this.config.model,
           messages: [
             { role: "system", content: params.systemPrompt },
-            { role: "user", content: params.userPrompt },
+            { role: "user", content: userContent },
           ],
           temperature: params.temperature ?? 0.3,
           max_tokens: params.maxTokens ?? 1000,

src/lib/ai/types.ts

@@ -114,11 +114,50 @@ export interface AIProvider {
   generateCompletion(params: CompletionParams): Promise<CompletionResult>;
 }
 
+/**
+ * v1.18.9 — a single vision input folded into the provider request. Used by
+ * the Lab-OCR extraction path: a photo of a paper report, or a PDF page.
+ * `dataBase64` is the raw file bytes, base64-encoded (no `data:` prefix —
+ * each client wraps it in the wire shape it needs). The blob is NEVER threaded
+ * into an `annotate()` meta or any log field (it is health data).
+ */
+export interface CompletionImage {
+  /** One of the sniffed image MIME types. */
+  mediaType: "image/jpeg" | "image/png" | "image/webp";
+  /** Raw file bytes, base64-encoded (no data-URL prefix). */
+  dataBase64: string;
+}
+
+/**
+ * v1.18.9 — a PDF document folded into the provider request. Only Anthropic
+ * accepts a native `document` block over the Messages API we use; the other
+ * clients ignore it (the OCR route gates PDFs to Anthropic providers).
+ */
+export interface CompletionDocument {
+  mediaType: "application/pdf";
+  /** Raw PDF bytes, base64-encoded (no data-URL prefix). */
+  dataBase64: string;
+}
+
 export interface CompletionParams {
   systemPrompt: string;
   userPrompt: string;
   temperature?: number;
   maxTokens?: number;
+  /**
+   * v1.18.9 — optional vision inputs (Lab-OCR). Additive: every text-only
+   * caller (Coach, status cards, insights, narratives) leaves these unset, so
+   * the existing wire shape and streaming path are untouched. A client that
+   * cannot read images/documents ignores the field. Treat the contents as
+   * UNTRUSTED data to transcribe — never as instructions.
+   */
+  images?: CompletionImage[];
+  /**
+   * v1.18.9 — optional PDF document input (Lab-OCR). Anthropic-only; other
+   * clients ignore it. Mutually informative with `images` — the OCR route
+   * sends one or the other depending on the upload type and provider.
+   */
+  documents?: CompletionDocument[];
   /**
    * Optional deterministic seed. Threaded onto the OpenAI and local
    * (Ollama / OpenAI-compatible) request bodies for reproducible output on

src/lib/ai/vision-capability.ts

@@ -0,0 +1,109 @@
+/**
+ * v1.18.9 — vision-capability resolution for the Lab-OCR ingestion path.
+ *
+ * The provider abstraction is text-only by default; only some providers +
+ * models can read an uploaded photo / PDF. This module is the single source of
+ * truth for "can the user's configured provider read images?", so the upload
+ * route and the capability probe endpoint agree.
+ *
+ * Capability is a function of (providerType, model), evaluated against the
+ * documented vision allowlists below. It is PURE and presence-only — it never
+ * decrypts a key, constructs a client, or probes liveness. A `true` here can
+ * still resolve to a dead provider at call time (revoked key, unreachable
+ * local host); the route surfaces that as an extract failure.
+ */
+
+/**
+ * Anthropic vision models. The Messages API `image` + `document` blocks are
+ * accepted by the Claude 3.5+ / 3.7 / 4 families. An older `claude-2*` /
+ * `claude-instant*` pin is text-only.
+ */
+function anthropicSupportsVision(model: string): boolean {
+  const m = model.toLowerCase();
+  return (
+    m.startsWith("claude-3-5") ||
+    m.startsWith("claude-3.5") ||
+    m.startsWith("claude-3-7") ||
+    m.startsWith("claude-3.7") ||
+    m.startsWith("claude-sonnet-4") ||
+    m.startsWith("claude-opus-4") ||
+    m.startsWith("claude-haiku-4")
+  );
+}
+
+/**
+ * OpenAI vision models (also covers the operator's admin key + Codex when it
+ * runs an OpenAI model — though Codex is excluded by `providerType`). `gpt-4o`,
+ * `gpt-4.1`, `gpt-4-turbo`, and the `o*` reasoning models read image content;
+ * `gpt-3.5*` and the legacy `gpt-4` (non-turbo) 400 on an image block.
+ */
+function openaiSupportsVision(model: string): boolean {
+  const m = model.toLowerCase();
+  return (
+    m.startsWith("gpt-4o") ||
+    m.startsWith("gpt-4.1") ||
+    m.startsWith("gpt-4-turbo") ||
+    m.startsWith("o1") ||
+    m.startsWith("o3") ||
+    m.startsWith("o4")
+  );
+}
+
+/**
+ * Logical provider tags the OCR path reasons about. Mirrors the
+ * `ProviderChainType` vocabulary plus the legacy `admin-key` single-provider
+ * tag `resolveProvider`'s admin fallback returns.
+ */
+export type VisionProviderType =
+  | "anthropic"
+  | "openai"
+  | "admin-openai"
+  | "admin-key"
+  | "local"
+  | "codex"
+  | "none";
+
+/**
+ * Pure capability check over a (providerType, model) pair.
+ *
+ *  - anthropic            → vision when the model is a 3.5+/4 family pin.
+ *  - openai / admin-*     → vision when the model is a gpt-4o/4.1/turbo/o* pin.
+ *  - local                → trust-by-default true: the operator opted in by
+ *                           configuring a self-hosted model and we cannot sniff
+ *                           its capabilities; a non-vision local model surfaces
+ *                           a clear extract failure rather than a silent gate.
+ *  - codex / none         → false (no vision over the Codex wire; nothing
+ *                           configured).
+ */
+export function supportsVisionForConfig(
+  providerType: VisionProviderType,
+  model: string | null,
+): boolean {
+  switch (providerType) {
+    case "anthropic":
+      return model ? anthropicSupportsVision(model) : false;
+    case "openai":
+    case "admin-openai":
+    case "admin-key":
+      return model ? openaiSupportsVision(model) : false;
+    case "local":
+      return true;
+    case "codex":
+    case "none":
+    default:
+      return false;
+  }
+}
+
+/**
+ * Whether PDFs are supported for a given provider. Only Anthropic accepts a
+ * native `document` block over the Messages API; OpenAI / local read images
+ * only (a server-side rasterizer is a deferred follow-up). The route uses this
+ * to reject a PDF upload with a clear "use a photo / configure Claude" message
+ * on a non-Anthropic provider.
+ */
+export function supportsPdfForProvider(
+  providerType: VisionProviderType,
+): boolean {
+  return providerType === "anthropic";
+}