feat(documents): rasterize PDFs so image-only-wire providers can read them

Only Anthropic accepts a native PDF block; codex and other image-only-wire
providers dropped it. Add a bounded, best-effort server-side rasterizer that
renders a PDF's pages to JPEG images those providers read as ordinary image
input, so a scanned/image-only PDF is legible on the auto-read path.

- @napi-rs/canvas promoted to a direct dependency; its musl prebuilts + the
  pdfjs loader pinned into the standalone trace (next.config.ts)
- rasterize-pdf.ts: lazy pdfjs import (keeps DOMMatrix out of the eager chunk),
  first 10 pages, 1600px long edge, JPEG q80, never throws
- prepareVisionInput rasterizes a PDF for a non-Anthropic vision provider and
  keeps the native PDF block for Anthropic; the document capability reports
  PDFs readable via raster so the UI offers the read

next.config.ts

@@ -254,6 +254,18 @@ const nextConfig: NextConfig = {
       // runtime image explicitly. pnpm keeps pdfjs-dist unhoisted under `.pnpm`.
       "./node_modules/.pnpm/pdf-parse@*/node_modules/pdf-parse/**",
       "./node_modules/.pnpm/pdfjs-dist@*/node_modules/pdfjs-dist/**",
+      // Document AI rasterization (`src/lib/documents/rasterize-pdf.ts`) renders
+      // a scanned/image-only PDF page to a JPEG via `@napi-rs/canvas` so an
+      // image-only-wire provider (codex/OAuth) can read it — the ambient
+      // auto-read path. `@napi-rs/canvas` loads a native `.node` binary that the
+      // Turbopack NFT tracer resolves dynamically and therefore misses, so pin
+      // the JS loader AND the two musl prebuilts (the Alpine runtime image is
+      // amd64 + arm64) into the standalone image explicitly. pnpm keeps them
+      // unhoisted under `.pnpm`. No cairo/pango apk is needed — the prebuilt
+      // `.node` is self-contained.
+      "./node_modules/.pnpm/@napi-rs+canvas@*/node_modules/@napi-rs/canvas/**",
+      "./node_modules/.pnpm/@napi-rs+canvas-linux-x64-musl@*/node_modules/@napi-rs/canvas-linux-x64-musl/**",
+      "./node_modules/.pnpm/@napi-rs+canvas-linux-arm64-musl@*/node_modules/@napi-rs/canvas-linux-arm64-musl/**",
     ],
   },
   // v1.4.34 IW-A — silence the Turbopack NFT trace warnings emitted

package.json

@@ -59,6 +59,7 @@
     "@dnd-kit/utilities": "^3.2.2",
     "@hookform/resolvers": "^5.4.0",
     "@modelcontextprotocol/sdk": "^1.29.0",
+    "@napi-rs/canvas": "0.1.80",
     "@node-rs/argon2": "^2.0.2",
     "@parse/node-apn": "^8.1.0",
     "@prisma/adapter-pg": "^7.8.0",
@@ -123,6 +124,7 @@
     "knip": "^6.16.1",
     "lefthook": "^2.1.9",
     "pdf-parse": "^2.4.5",
+    "pdfjs-dist": "5.4.296",
     "pg": "^8.22.0",
     "prettier": "^3.9.1",
     "prettier-plugin-tailwindcss": "^0.8.0",

pnpm-lock.yaml

@@ -30,6 +30,9 @@ importers:
       '@modelcontextprotocol/sdk':
         specifier: ^1.29.0
         version: 1.29.0(zod@4.4.3)
+      '@napi-rs/canvas':
+        specifier: 0.1.80
+        version: 0.1.80
       '@node-rs/argon2':
         specifier: ^2.0.2
         version: 2.0.2
@@ -217,6 +220,9 @@ importers:
       pdf-parse:
         specifier: ^2.4.5
         version: 2.4.5
+      pdfjs-dist:
+        specifier: 5.4.296
+        version: 5.4.296
       pg:
         specifier: ^8.22.0
         version: 8.22.0

src/app/api/documents/inbound/[id]/index/route.ts

@@ -184,7 +184,7 @@ async function handleVisionIndex(
   );
   if (!rl.allowed) return rateLimited(rl);
 
-  const vision = prepareVisionInput(document, pick.pdfSupported);
+  const vision = await prepareVisionInput(document, pick.pdfSupported);
   if (!vision.ok) {
     if (vision.reason === "pdfNeedsAnthropic") {
       return apiError(

src/app/api/documents/inbound/[id]/suggest/route.ts

@@ -233,7 +233,7 @@ async function handleVisionSuggest(
   );
   if (!rl.allowed) return rateLimited(rl);
 
-  const vision = prepareVisionInput(document, pick.pdfSupported);
+  const vision = await prepareVisionInput(document, pick.pdfSupported);
   if (!vision.ok) {
     if (vision.reason === "pdfNeedsAnthropic") {
       return apiError(

src/app/api/documents/inbound/[id]/summary/route.ts

@@ -258,7 +258,7 @@ async function handleVisionSummary(
   );
   if (!rl.allowed) return rateLimited(rl);
 
-  const vision = prepareVisionInput(document, pick.pdfSupported);
+  const vision = await prepareVisionInput(document, pick.pdfSupported);
   if (!vision.ok) {
     if (vision.reason === "pdfNeedsAnthropic") {
       return apiError(

src/lib/documents/__tests__/rasterize-pdf.test.ts

@@ -0,0 +1,104 @@
+import { Buffer } from "node:buffer";
+
+import { describe, expect, it, vi } from "vitest";
+
+vi.mock("@/lib/logging/context", () => ({ annotate: vi.fn() }));
+
+import { RASTER_MAX_PAGES, rasterizePdf } from "../rasterize-pdf";
+
+/**
+ * Rasterization renders a PDF's pages to bounded JPEG images. Pins: a text-layer
+ * PDF and a synthetic image-only (graphics-only, no text layer) PDF both render
+ * to JPEG page images; the page cap is honoured; malformed input degrades to
+ * `{ ok: false }` and never throws.
+ *
+ * The fixtures are built inline (a tiny hand-assembled PDF with a correct xref)
+ * so the test carries no binary blobs.
+ */
+function buildPdf(pageCount: number, withText: boolean): Buffer {
+  const pageNums: number[] = [];
+  const perPage: { pageNum: number; contentNum: number; content: string }[] =
+    [];
+  let n = 4;
+  for (let p = 0; p < pageCount; p++) {
+    const pageNum = n++;
+    const contentNum = n++;
+    pageNums.push(pageNum);
+    const content = withText
+      ? `BT /F1 24 Tf 40 120 Td (Page ${p + 1} lab report glucose creatinine) Tj ET`
+      : `0 0 1 rg 40 40 120 120 re f`;
+    perPage.push({ pageNum, contentNum, content });
+  }
+
+  const bodies: string[] = [];
+  bodies[1] = `<< /Type /Catalog /Pages 2 0 R >>`;
+  bodies[2] = `<< /Type /Pages /Kids [${pageNums
+    .map((x) => `${x} 0 R`)
+    .join(" ")}] /Count ${pageCount} >>`;
+  bodies[3] = `<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>`;
+  for (const { pageNum, contentNum, content } of perPage) {
+    bodies[pageNum] =
+      `<< /Type /Page /Parent 2 0 R /MediaBox [0 0 200 200] /Resources << /Font << /F1 3 0 R >> >> /Contents ${contentNum} 0 R >>`;
+    bodies[contentNum] =
+      `<< /Length ${Buffer.byteLength(content, "latin1")} >>\nstream\n${content}\nendstream`;
+  }
+
+  const total = n - 1;
+  let out = "%PDF-1.4\n";
+  const offsets: number[] = [];
+  for (let i = 1; i <= total; i++) {
+    offsets[i] = Buffer.byteLength(out, "latin1");
+    out += `${i} 0 obj\n${bodies[i]}\nendobj\n`;
+  }
+  const xrefOffset = Buffer.byteLength(out, "latin1");
+  out += `xref\n0 ${total + 1}\n0000000000 65535 f \n`;
+  for (let i = 1; i <= total; i++) {
+    out += `${String(offsets[i]).padStart(10, "0")} 00000 n \n`;
+  }
+  out += `trailer\n<< /Size ${total + 1} /Root 1 0 R >>\nstartxref\n${xrefOffset}\n%%EOF`;
+  return Buffer.from(out, "latin1");
+}
+
+/** JPEG magic bytes. */
+function isJpeg(base64: string): boolean {
+  const head = Buffer.from(base64, "base64").subarray(0, 2).toString("hex");
+  return head === "ffd8";
+}
+
+describe("rasterizePdf", () => {
+  it("renders a text-layer PDF's pages to JPEG images", async () => {
+    const result = await rasterizePdf(buildPdf(3, true));
+    expect(result.ok).toBe(true);
+    if (!result.ok) return;
+    expect(result.images).toHaveLength(3);
+    for (const img of result.images) {
+      expect(img.mediaType).toBe("image/jpeg");
+      expect(isJpeg(img.dataBase64)).toBe(true);
+    }
+  });
+
+  it("renders a synthetic image-only (no text layer) PDF", async () => {
+    const result = await rasterizePdf(buildPdf(1, false));
+    expect(result.ok).toBe(true);
+    if (!result.ok) return;
+    expect(result.images).toHaveLength(1);
+    expect(isJpeg(result.images[0]!.dataBase64)).toBe(true);
+  });
+
+  it("caps the number of rendered pages at RASTER_MAX_PAGES", async () => {
+    const result = await rasterizePdf(buildPdf(RASTER_MAX_PAGES + 4, true));
+    expect(result.ok).toBe(true);
+    if (!result.ok) return;
+    expect(result.images).toHaveLength(RASTER_MAX_PAGES);
+  });
+
+  it("returns { ok: false } and never throws on malformed input", async () => {
+    await expect(
+      rasterizePdf(Buffer.from("this is not a pdf")),
+    ).resolves.toEqual({ ok: false });
+  });
+
+  it("returns { ok: false } on an empty buffer", async () => {
+    await expect(rasterizePdf(Buffer.alloc(0))).resolves.toEqual({ ok: false });
+  });
+});

src/lib/documents/ai-route-support.ts

@@ -10,6 +10,7 @@
 import { Buffer } from "node:buffer";
 
 import { prisma } from "@/lib/db";
+import { rasterizePdf } from "@/lib/documents/rasterize-pdf";
 import { decryptDocumentContent } from "@/lib/documents/store";
 import { detectOcrMimeType } from "@/lib/labs/ocr-upload";
 
@@ -75,13 +76,24 @@ export type VisionInput =
 /**
  * Decrypt the stored original, re-derive its MIME from the bytes (never trust
  * the stored label for a provider call), and split it into the images/documents
- * arrays a vision call takes. Fails closed on a decrypt error and rejects a PDF
- * on a provider that cannot read PDFs natively.
+ * arrays a vision call takes. Fails closed on a decrypt error.
+ *
+ * PDF handling by provider capability:
+ *   - `pdfSupported` (Anthropic) → native `document` block, highest fidelity.
+ *   - otherwise (codex / any image-only-wire provider) → rasterize the pages to
+ *     JPEG `input_image` parts so the provider can still read the PDF. Egress is
+ *     already authorised upstream (the consent gate / the `documentsAutoAiRead`
+ *     toggle) before this is reached. If rasterization fails (malformed /
+ *     encrypted / unrenderable), fall back to the shipped `pdfNeedsAnthropic`
+ *     outcome so the job degrades to the local text-layer path and an
+ *     interactive route surfaces the same actionable error.
+ *
+ * Images (jpg/png/webp) flow straight to `input_image`, unchanged.
  */
-export function prepareVisionInput(
+export async function prepareVisionInput(
   document: LoadedDocument,
   pdfSupported: boolean,
-): VisionInput {
+): Promise<VisionInput> {
   let buffer: Buffer;
   try {
     buffer = decryptDocumentContent(
@@ -94,17 +106,31 @@ export function prepareVisionInput(
 
   const mime = detectOcrMimeType(buffer);
   if (!mime) return { ok: false, reason: "fileType" };
-  if (mime === "application/pdf" && !pdfSupported) {
+
+  if (mime === "application/pdf") {
+    if (pdfSupported) {
+      return {
+        ok: true,
+        images: [],
+        documents: [
+          {
+            mediaType: "application/pdf",
+            dataBase64: buffer.toString("base64"),
+          },
+        ],
+      };
+    }
+    // Non-Anthropic vision provider — render the PDF pages to images it can read.
+    const raster = await rasterizePdf(buffer);
+    if (raster.ok) {
+      return { ok: true, images: raster.images, documents: [] };
+    }
     return { ok: false, reason: "pdfNeedsAnthropic" };
   }
 
-  const dataBase64 = buffer.toString("base64");
-  if (mime === "application/pdf") {
-    return {
-      ok: true,
-      images: [],
-      documents: [{ mediaType: "application/pdf", dataBase64 }],
-    };
-  }
-  return { ok: true, images: [{ mediaType: mime, dataBase64 }], documents: [] };
+  return {
+    ok: true,
+    images: [{ mediaType: mime, dataBase64: buffer.toString("base64") }],
+    documents: [],
+  };
 }

src/lib/documents/index-document.ts

@@ -128,7 +128,7 @@ async function tryProviderIndex(
   const { pick } = provider;
   if (!pick || !provider.consentOk) return null;
 
-  const vision = prepareVisionInput(document, pick.pdfSupported);
+  const vision = await prepareVisionInput(document, pick.pdfSupported);
   if (!vision.ok) {
     // A decrypt failure is terminal (local would fail the same way); anything
     // else (fileType / pdfNeedsAnthropic) falls through to the local path — a

src/lib/documents/provider-order.ts

@@ -25,6 +25,7 @@
  * external egress from happening without an active receipt.
  */
 import { isExternalDocumentEgress } from "@/lib/ai/consent-guard";
+import { RASTERIZATION_AVAILABLE } from "@/lib/documents/rasterize-pdf";
 import {
   resolveTextProvider,
   resolveVisionProvider,
@@ -119,13 +120,16 @@ export async function resolveDocumentAiCapability(
     await resolveDocumentVisionProvider(userId);
 
   // A vision-capable provider is available — the read runs directly over the
-  // stored original. Egress follows the picked provider.
+  // stored original. Egress follows the picked provider. PDFs are readable
+  // whenever the picked provider natively supports them (Anthropic) OR the
+  // server-side rasterizer is available (every other vision provider reads a
+  // PDF via rasterized page images), so the UI offers a PDF read on codex too.
   if (pick) {
     return {
       available: true,
       mode: "vision",
       reason: null,
-      pdfSupported: pick.pdfSupported,
+      pdfSupported: pick.pdfSupported || RASTERIZATION_AVAILABLE,
       egress: documentEgressClass(pick.providerType),
     };
   }

src/lib/documents/rasterize-pdf.ts

@@ -0,0 +1,191 @@
+/**
+ * Server-side PDF rasterization for the document vault.
+ *
+ * The OAuth/subscription wire (codex) — and every other image-only-wire provider
+ * — cannot receive a native PDF `document` block; only Anthropic accepts one.
+ * This module renders a PDF's pages to raster JPEG images so those providers can
+ * still READ a scanned/image-only PDF: each page image goes out as an ordinary
+ * `input_image` part the codex client already handles, with no client-interface
+ * change. Only invoked (on the auto-read path) when the toggle authorises the
+ * external egress and the picked provider is NOT Anthropic (Anthropic keeps its
+ * higher-fidelity native PDF block; a text-layer PDF is still read locally for
+ * free before we ever rasterize).
+ *
+ * pdfjs is loaded LAZILY via a runtime `import()` — its module top-level
+ * references the browser-only `DOMMatrix` global, so a STATIC import would make
+ * the Turbopack server chunk evaluate that reference at instantiation and throw
+ * `DOMMatrix is not defined`, taking down every route/worker sharing the chunk.
+ * The lazy import (the exact pattern `local-extract.ts` documents) keeps pdfjs +
+ * `@napi-rs/canvas` (its Node canvas backend) out of the eager chunk. Rendering
+ * is pure local compute — no network, nothing written to disk.
+ *
+ * Bounds (denial-of-service + vision token cost): render at most the first
+ * `RASTER_MAX_PAGES` pages, each capped to `RASTER_TARGET_LONG_EDGE` on its long
+ * edge (never upscaled), JPEG at `RASTER_JPEG_QUALITY`. A discharge letter / lab
+ * is 1-4 pages; the cap keeps a pathological 200-page PDF from draining the
+ * subscription allowance in one job.
+ *
+ * NEVER throws: a malformed / encrypted / unrenderable PDF, a missing native
+ * binary, or any render error resolves to `{ ok: false }`, and the caller falls
+ * back to the local text-layer path or leaves the document un-indexed. This
+ * preserves the "a bad document never aborts an upload/batch" contract.
+ */
+import { Buffer } from "node:buffer";
+
+import { annotate } from "@/lib/logging/context";
+
+/**
+ * Whether the PDF rasterizer is available in this build. The native
+ * `@napi-rs/canvas` binary is compiled in and traced into the standalone image
+ * (see `next.config.ts`), so rasterization is a standing capability — every
+ * vision provider can read a PDF (Anthropic natively, all others via raster).
+ * The document capability DTO reads this so the UI offers a PDF read for a
+ * non-Anthropic provider; a runtime render failure still degrades gracefully
+ * (`prepareVisionInput` → `pdfNeedsAnthropic` → local text or un-indexed).
+ */
+export const RASTERIZATION_AVAILABLE = true;
+
+/** Render at most the first N pages — the token-cost / DoS governor. */
+export const RASTER_MAX_PAGES = 10;
+
+/**
+ * Longest-edge pixel cap per page. Tighter than the client upload downscale
+ * ceiling because these feed a token-metered vision model; ~1600px is legible
+ * for a document read without blowing up the tile count.
+ */
+export const RASTER_TARGET_LONG_EDGE = 1600;
+
+/** JPEG quality (0-100). ~80 keeps text legible at a fraction of PNG's bytes. */
+export const RASTER_JPEG_QUALITY = 80;
+
+/** One rasterized page, shaped as a vision `input_image` part. */
+export interface RasterImage {
+  mediaType: "image/jpeg";
+  dataBase64: string;
+}
+
+/** Best-effort rasterization outcome — never an exception. */
+export type RasterResult = { ok: true; images: RasterImage[] } | { ok: false };
+
+// Minimal structural types for the slice of the pdfjs API this module uses. A
+// type-only shape (erased at build) so we never eagerly evaluate the pdfjs
+// module for its types; the real module is pulled at runtime via `import()`.
+interface PdfViewport {
+  width: number;
+  height: number;
+}
+interface PdfCanvas {
+  width: number;
+  height: number;
+  toBuffer(mime: "image/jpeg", quality?: number): Buffer;
+}
+interface PdfCanvasAndContext {
+  canvas: PdfCanvas;
+  context: unknown;
+}
+interface PdfCanvasFactory {
+  create(width: number, height: number): PdfCanvasAndContext;
+  destroy?(target: PdfCanvasAndContext): void;
+}
+interface PdfPage {
+  getViewport(opts: { scale: number }): PdfViewport;
+  render(opts: {
+    canvasContext: unknown;
+    viewport: PdfViewport;
+    canvas: unknown;
+  }): { promise: Promise<void> };
+  cleanup(): void;
+}
+interface PdfDocument {
+  numPages: number;
+  canvasFactory: PdfCanvasFactory;
+  getPage(pageNumber: number): Promise<PdfPage>;
+  destroy(): Promise<void>;
+}
+interface PdfjsModule {
+  getDocument(opts: {
+    data: Uint8Array;
+    verbosity?: number;
+    isEvalSupported?: boolean;
+    useSystemFonts?: boolean;
+  }): { promise: Promise<PdfDocument> };
+}
+
+/**
+ * Render the first `RASTER_MAX_PAGES` pages of a PDF to bounded JPEG images.
+ * Returns `{ ok: false }` on any failure (malformed PDF, render throw, missing
+ * binary, or zero renderable pages) — the caller degrades to local text or
+ * un-indexed. Never throws.
+ */
+export async function rasterizePdf(buffer: Buffer): Promise<RasterResult> {
+  let doc: PdfDocument | null = null;
+  try {
+    // Lazy import: keeps pdfjs (DOMMatrix) + @napi-rs/canvas out of the eager
+    // server chunk. Resolved at first rasterization, then module-cached.
+    const pdfjs =
+      (await import("pdfjs-dist/legacy/build/pdf.mjs")) as unknown as PdfjsModule;
+
+    const task = pdfjs.getDocument({
+      data: new Uint8Array(buffer),
+      verbosity: 0, // VerbosityLevel.ERRORS — no console spam on odd PDFs.
+      isEvalSupported: false, // never eval font programs (defence in depth).
+      useSystemFonts: true,
+    });
+    doc = await task.promise;
+
+    const pageCount = Math.min(doc.numPages, RASTER_MAX_PAGES);
+    const images: RasterImage[] = [];
+
+    for (let pageNumber = 1; pageNumber <= pageCount; pageNumber++) {
+      const page = await doc.getPage(pageNumber);
+      try {
+        const base = page.getViewport({ scale: 1 });
+        const longEdge = Math.max(base.width, base.height);
+        // Cap the long edge, never upscale (scale ≤ 1).
+        const scale =
+          longEdge > 0 ? Math.min(1, RASTER_TARGET_LONG_EDGE / longEdge) : 1;
+        const viewport = page.getViewport({ scale });
+
+        const factory = doc.canvasFactory;
+        const cc = factory.create(
+          Math.ceil(viewport.width),
+          Math.ceil(viewport.height),
+        );
+        await page.render({
+          canvasContext: cc.context,
+          viewport,
+          canvas: cc.canvas,
+        }).promise;
+        const jpeg = cc.canvas.toBuffer("image/jpeg", RASTER_JPEG_QUALITY);
+        images.push({
+          mediaType: "image/jpeg",
+          dataBase64: jpeg.toString("base64"),
+        });
+        factory.destroy?.(cc);
+      } finally {
+        page.cleanup();
+      }
+    }
+
+    if (images.length === 0) return { ok: false };
+    annotate({
+      action: { name: "documents.rasterize.ok" },
+      meta: { pages: images.length, cappedFrom: doc.numPages },
+    });
+    return { ok: true, images };
+  } catch (err) {
+    annotate({
+      action: { name: "documents.rasterize.failed" },
+      meta: { reason: err instanceof Error ? err.name : "unknown" },
+    });
+    return { ok: false };
+  } finally {
+    if (doc) {
+      try {
+        await doc.destroy();
+      } catch {
+        // Best-effort teardown; a destroy failure must not mask the result.
+      }
+    }
+  }
+}

src/lib/jobs/document-content-index-backfill.ts

@@ -137,7 +137,7 @@ export async function runContentIndexBackfillForUser(
         await reconcileSpend(userId, reservation.reserved, 0, dateKey);
         continue;
       }
-      const vision = prepareVisionInput(document, pick.pdfSupported);
+      const vision = await prepareVisionInput(document, pick.pdfSupported);
       if (!vision.ok) {
         // Not vision-indexable (should be filtered by MIME, but guard) — refund
         // and move the cursor past it so the walk keeps converging.