From d2451021bc6620e3a1a8d424eabbaea763911d28 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Thu, 9 Jul 2026 10:49:17 +0200 Subject: [PATCH] feat(documents): rasterize PDFs so image-only-wire providers can read them Only Anthropic accepts a native PDF block; codex and other image-only-wire providers dropped it. Add a bounded, best-effort server-side rasterizer that renders a PDF's pages to JPEG images those providers read as ordinary image input, so a scanned/image-only PDF is legible on the auto-read path. - @napi-rs/canvas promoted to a direct dependency; its musl prebuilts + the pdfjs loader pinned into the standalone trace (next.config.ts) - rasterize-pdf.ts: lazy pdfjs import (keeps DOMMatrix out of the eager chunk), first 10 pages, 1600px long edge, JPEG q80, never throws - prepareVisionInput rasterizes a PDF for a non-Anthropic vision provider and keeps the native PDF block for Anthropic; the document capability reports PDFs readable via raster so the UI offers the read --- next.config.ts | 12 ++ package.json | 2 + pnpm-lock.yaml | 6 + .../api/documents/inbound/[id]/index/route.ts | 2 +- .../documents/inbound/[id]/suggest/route.ts | 2 +- .../documents/inbound/[id]/summary/route.ts | 2 +- .../documents/__tests__/rasterize-pdf.test.ts | 104 ++++++++++ src/lib/documents/ai-route-support.ts | 54 +++-- src/lib/documents/index-document.ts | 2 +- src/lib/documents/provider-order.ts | 8 +- src/lib/documents/rasterize-pdf.ts | 191 ++++++++++++++++++ .../jobs/document-content-index-backfill.ts | 2 +- 12 files changed, 366 insertions(+), 21 deletions(-) create mode 100644 src/lib/documents/__tests__/rasterize-pdf.test.ts create mode 100644 src/lib/documents/rasterize-pdf.ts diff --git a/next.config.ts b/next.config.ts index 63a17d195..fc2280969 100644 --- a/next.config.ts +++ b/next.config.ts @@ -254,6 +254,18 @@ const nextConfig: NextConfig = { // runtime image explicitly. pnpm keeps pdfjs-dist unhoisted under `.pnpm`. "./node_modules/.pnpm/pdf-parse@*/node_modules/pdf-parse/**", "./node_modules/.pnpm/pdfjs-dist@*/node_modules/pdfjs-dist/**", + // Document AI rasterization (`src/lib/documents/rasterize-pdf.ts`) renders + // a scanned/image-only PDF page to a JPEG via `@napi-rs/canvas` so an + // image-only-wire provider (codex/OAuth) can read it — the ambient + // auto-read path. `@napi-rs/canvas` loads a native `.node` binary that the + // Turbopack NFT tracer resolves dynamically and therefore misses, so pin + // the JS loader AND the two musl prebuilts (the Alpine runtime image is + // amd64 + arm64) into the standalone image explicitly. pnpm keeps them + // unhoisted under `.pnpm`. No cairo/pango apk is needed — the prebuilt + // `.node` is self-contained. + "./node_modules/.pnpm/@napi-rs+canvas@*/node_modules/@napi-rs/canvas/**", + "./node_modules/.pnpm/@napi-rs+canvas-linux-x64-musl@*/node_modules/@napi-rs/canvas-linux-x64-musl/**", + "./node_modules/.pnpm/@napi-rs+canvas-linux-arm64-musl@*/node_modules/@napi-rs/canvas-linux-arm64-musl/**", ], }, // v1.4.34 IW-A — silence the Turbopack NFT trace warnings emitted diff --git a/package.json b/package.json index 250ed1262..909cddd16 100644 --- a/package.json +++ b/package.json @@ -59,6 +59,7 @@ "@dnd-kit/utilities": "^3.2.2", "@hookform/resolvers": "^5.4.0", "@modelcontextprotocol/sdk": "^1.29.0", + "@napi-rs/canvas": "0.1.80", "@node-rs/argon2": "^2.0.2", "@parse/node-apn": "^8.1.0", "@prisma/adapter-pg": "^7.8.0", @@ -123,6 +124,7 @@ "knip": "^6.16.1", "lefthook": "^2.1.9", "pdf-parse": "^2.4.5", + "pdfjs-dist": "5.4.296", "pg": "^8.22.0", "prettier": "^3.9.1", "prettier-plugin-tailwindcss": "^0.8.0", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 4772a0b7e..6c75a0116 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -30,6 +30,9 @@ importers: '@modelcontextprotocol/sdk': specifier: ^1.29.0 version: 1.29.0(zod@4.4.3) + '@napi-rs/canvas': + specifier: 0.1.80 + version: 0.1.80 '@node-rs/argon2': specifier: ^2.0.2 version: 2.0.2 @@ -217,6 +220,9 @@ importers: pdf-parse: specifier: ^2.4.5 version: 2.4.5 + pdfjs-dist: + specifier: 5.4.296 + version: 5.4.296 pg: specifier: ^8.22.0 version: 8.22.0 diff --git a/src/app/api/documents/inbound/[id]/index/route.ts b/src/app/api/documents/inbound/[id]/index/route.ts index 0d318a235..665a93d05 100644 --- a/src/app/api/documents/inbound/[id]/index/route.ts +++ b/src/app/api/documents/inbound/[id]/index/route.ts @@ -184,7 +184,7 @@ async function handleVisionIndex( ); if (!rl.allowed) return rateLimited(rl); - const vision = prepareVisionInput(document, pick.pdfSupported); + const vision = await prepareVisionInput(document, pick.pdfSupported); if (!vision.ok) { if (vision.reason === "pdfNeedsAnthropic") { return apiError( diff --git a/src/app/api/documents/inbound/[id]/suggest/route.ts b/src/app/api/documents/inbound/[id]/suggest/route.ts index e0670d3f7..79a939fb2 100644 --- a/src/app/api/documents/inbound/[id]/suggest/route.ts +++ b/src/app/api/documents/inbound/[id]/suggest/route.ts @@ -233,7 +233,7 @@ async function handleVisionSuggest( ); if (!rl.allowed) return rateLimited(rl); - const vision = prepareVisionInput(document, pick.pdfSupported); + const vision = await prepareVisionInput(document, pick.pdfSupported); if (!vision.ok) { if (vision.reason === "pdfNeedsAnthropic") { return apiError( diff --git a/src/app/api/documents/inbound/[id]/summary/route.ts b/src/app/api/documents/inbound/[id]/summary/route.ts index 6d09219cb..dfce5bff3 100644 --- a/src/app/api/documents/inbound/[id]/summary/route.ts +++ b/src/app/api/documents/inbound/[id]/summary/route.ts @@ -258,7 +258,7 @@ async function handleVisionSummary( ); if (!rl.allowed) return rateLimited(rl); - const vision = prepareVisionInput(document, pick.pdfSupported); + const vision = await prepareVisionInput(document, pick.pdfSupported); if (!vision.ok) { if (vision.reason === "pdfNeedsAnthropic") { return apiError( diff --git a/src/lib/documents/__tests__/rasterize-pdf.test.ts b/src/lib/documents/__tests__/rasterize-pdf.test.ts new file mode 100644 index 000000000..547661932 --- /dev/null +++ b/src/lib/documents/__tests__/rasterize-pdf.test.ts @@ -0,0 +1,104 @@ +import { Buffer } from "node:buffer"; + +import { describe, expect, it, vi } from "vitest"; + +vi.mock("@/lib/logging/context", () => ({ annotate: vi.fn() })); + +import { RASTER_MAX_PAGES, rasterizePdf } from "../rasterize-pdf"; + +/** + * Rasterization renders a PDF's pages to bounded JPEG images. Pins: a text-layer + * PDF and a synthetic image-only (graphics-only, no text layer) PDF both render + * to JPEG page images; the page cap is honoured; malformed input degrades to + * `{ ok: false }` and never throws. + * + * The fixtures are built inline (a tiny hand-assembled PDF with a correct xref) + * so the test carries no binary blobs. + */ +function buildPdf(pageCount: number, withText: boolean): Buffer { + const pageNums: number[] = []; + const perPage: { pageNum: number; contentNum: number; content: string }[] = + []; + let n = 4; + for (let p = 0; p < pageCount; p++) { + const pageNum = n++; + const contentNum = n++; + pageNums.push(pageNum); + const content = withText + ? `BT /F1 24 Tf 40 120 Td (Page ${p + 1} lab report glucose creatinine) Tj ET` + : `0 0 1 rg 40 40 120 120 re f`; + perPage.push({ pageNum, contentNum, content }); + } + + const bodies: string[] = []; + bodies[1] = `<< /Type /Catalog /Pages 2 0 R >>`; + bodies[2] = `<< /Type /Pages /Kids [${pageNums + .map((x) => `${x} 0 R`) + .join(" ")}] /Count ${pageCount} >>`; + bodies[3] = `<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>`; + for (const { pageNum, contentNum, content } of perPage) { + bodies[pageNum] = + `<< /Type /Page /Parent 2 0 R /MediaBox [0 0 200 200] /Resources << /Font << /F1 3 0 R >> >> /Contents ${contentNum} 0 R >>`; + bodies[contentNum] = + `<< /Length ${Buffer.byteLength(content, "latin1")} >>\nstream\n${content}\nendstream`; + } + + const total = n - 1; + let out = "%PDF-1.4\n"; + const offsets: number[] = []; + for (let i = 1; i <= total; i++) { + offsets[i] = Buffer.byteLength(out, "latin1"); + out += `${i} 0 obj\n${bodies[i]}\nendobj\n`; + } + const xrefOffset = Buffer.byteLength(out, "latin1"); + out += `xref\n0 ${total + 1}\n0000000000 65535 f \n`; + for (let i = 1; i <= total; i++) { + out += `${String(offsets[i]).padStart(10, "0")} 00000 n \n`; + } + out += `trailer\n<< /Size ${total + 1} /Root 1 0 R >>\nstartxref\n${xrefOffset}\n%%EOF`; + return Buffer.from(out, "latin1"); +} + +/** JPEG magic bytes. */ +function isJpeg(base64: string): boolean { + const head = Buffer.from(base64, "base64").subarray(0, 2).toString("hex"); + return head === "ffd8"; +} + +describe("rasterizePdf", () => { + it("renders a text-layer PDF's pages to JPEG images", async () => { + const result = await rasterizePdf(buildPdf(3, true)); + expect(result.ok).toBe(true); + if (!result.ok) return; + expect(result.images).toHaveLength(3); + for (const img of result.images) { + expect(img.mediaType).toBe("image/jpeg"); + expect(isJpeg(img.dataBase64)).toBe(true); + } + }); + + it("renders a synthetic image-only (no text layer) PDF", async () => { + const result = await rasterizePdf(buildPdf(1, false)); + expect(result.ok).toBe(true); + if (!result.ok) return; + expect(result.images).toHaveLength(1); + expect(isJpeg(result.images[0]!.dataBase64)).toBe(true); + }); + + it("caps the number of rendered pages at RASTER_MAX_PAGES", async () => { + const result = await rasterizePdf(buildPdf(RASTER_MAX_PAGES + 4, true)); + expect(result.ok).toBe(true); + if (!result.ok) return; + expect(result.images).toHaveLength(RASTER_MAX_PAGES); + }); + + it("returns { ok: false } and never throws on malformed input", async () => { + await expect( + rasterizePdf(Buffer.from("this is not a pdf")), + ).resolves.toEqual({ ok: false }); + }); + + it("returns { ok: false } on an empty buffer", async () => { + await expect(rasterizePdf(Buffer.alloc(0))).resolves.toEqual({ ok: false }); + }); +}); diff --git a/src/lib/documents/ai-route-support.ts b/src/lib/documents/ai-route-support.ts index 64cbb6221..a974ad5ab 100644 --- a/src/lib/documents/ai-route-support.ts +++ b/src/lib/documents/ai-route-support.ts @@ -10,6 +10,7 @@ import { Buffer } from "node:buffer"; import { prisma } from "@/lib/db"; +import { rasterizePdf } from "@/lib/documents/rasterize-pdf"; import { decryptDocumentContent } from "@/lib/documents/store"; import { detectOcrMimeType } from "@/lib/labs/ocr-upload"; @@ -75,13 +76,24 @@ export type VisionInput = /** * Decrypt the stored original, re-derive its MIME from the bytes (never trust * the stored label for a provider call), and split it into the images/documents - * arrays a vision call takes. Fails closed on a decrypt error and rejects a PDF - * on a provider that cannot read PDFs natively. + * arrays a vision call takes. Fails closed on a decrypt error. + * + * PDF handling by provider capability: + * - `pdfSupported` (Anthropic) → native `document` block, highest fidelity. + * - otherwise (codex / any image-only-wire provider) → rasterize the pages to + * JPEG `input_image` parts so the provider can still read the PDF. Egress is + * already authorised upstream (the consent gate / the `documentsAutoAiRead` + * toggle) before this is reached. If rasterization fails (malformed / + * encrypted / unrenderable), fall back to the shipped `pdfNeedsAnthropic` + * outcome so the job degrades to the local text-layer path and an + * interactive route surfaces the same actionable error. + * + * Images (jpg/png/webp) flow straight to `input_image`, unchanged. */ -export function prepareVisionInput( +export async function prepareVisionInput( document: LoadedDocument, pdfSupported: boolean, -): VisionInput { +): Promise { let buffer: Buffer; try { buffer = decryptDocumentContent( @@ -94,17 +106,31 @@ export function prepareVisionInput( const mime = detectOcrMimeType(buffer); if (!mime) return { ok: false, reason: "fileType" }; - if (mime === "application/pdf" && !pdfSupported) { + + if (mime === "application/pdf") { + if (pdfSupported) { + return { + ok: true, + images: [], + documents: [ + { + mediaType: "application/pdf", + dataBase64: buffer.toString("base64"), + }, + ], + }; + } + // Non-Anthropic vision provider — render the PDF pages to images it can read. + const raster = await rasterizePdf(buffer); + if (raster.ok) { + return { ok: true, images: raster.images, documents: [] }; + } return { ok: false, reason: "pdfNeedsAnthropic" }; } - const dataBase64 = buffer.toString("base64"); - if (mime === "application/pdf") { - return { - ok: true, - images: [], - documents: [{ mediaType: "application/pdf", dataBase64 }], - }; - } - return { ok: true, images: [{ mediaType: mime, dataBase64 }], documents: [] }; + return { + ok: true, + images: [{ mediaType: mime, dataBase64: buffer.toString("base64") }], + documents: [], + }; } diff --git a/src/lib/documents/index-document.ts b/src/lib/documents/index-document.ts index 193e3dd6b..319dae074 100644 --- a/src/lib/documents/index-document.ts +++ b/src/lib/documents/index-document.ts @@ -128,7 +128,7 @@ async function tryProviderIndex( const { pick } = provider; if (!pick || !provider.consentOk) return null; - const vision = prepareVisionInput(document, pick.pdfSupported); + const vision = await prepareVisionInput(document, pick.pdfSupported); if (!vision.ok) { // A decrypt failure is terminal (local would fail the same way); anything // else (fileType / pdfNeedsAnthropic) falls through to the local path — a diff --git a/src/lib/documents/provider-order.ts b/src/lib/documents/provider-order.ts index e27678d50..5107dcef3 100644 --- a/src/lib/documents/provider-order.ts +++ b/src/lib/documents/provider-order.ts @@ -25,6 +25,7 @@ * external egress from happening without an active receipt. */ import { isExternalDocumentEgress } from "@/lib/ai/consent-guard"; +import { RASTERIZATION_AVAILABLE } from "@/lib/documents/rasterize-pdf"; import { resolveTextProvider, resolveVisionProvider, @@ -119,13 +120,16 @@ export async function resolveDocumentAiCapability( await resolveDocumentVisionProvider(userId); // A vision-capable provider is available — the read runs directly over the - // stored original. Egress follows the picked provider. + // stored original. Egress follows the picked provider. PDFs are readable + // whenever the picked provider natively supports them (Anthropic) OR the + // server-side rasterizer is available (every other vision provider reads a + // PDF via rasterized page images), so the UI offers a PDF read on codex too. if (pick) { return { available: true, mode: "vision", reason: null, - pdfSupported: pick.pdfSupported, + pdfSupported: pick.pdfSupported || RASTERIZATION_AVAILABLE, egress: documentEgressClass(pick.providerType), }; } diff --git a/src/lib/documents/rasterize-pdf.ts b/src/lib/documents/rasterize-pdf.ts new file mode 100644 index 000000000..b880441c9 --- /dev/null +++ b/src/lib/documents/rasterize-pdf.ts @@ -0,0 +1,191 @@ +/** + * Server-side PDF rasterization for the document vault. + * + * The OAuth/subscription wire (codex) — and every other image-only-wire provider + * — cannot receive a native PDF `document` block; only Anthropic accepts one. + * This module renders a PDF's pages to raster JPEG images so those providers can + * still READ a scanned/image-only PDF: each page image goes out as an ordinary + * `input_image` part the codex client already handles, with no client-interface + * change. Only invoked (on the auto-read path) when the toggle authorises the + * external egress and the picked provider is NOT Anthropic (Anthropic keeps its + * higher-fidelity native PDF block; a text-layer PDF is still read locally for + * free before we ever rasterize). + * + * pdfjs is loaded LAZILY via a runtime `import()` — its module top-level + * references the browser-only `DOMMatrix` global, so a STATIC import would make + * the Turbopack server chunk evaluate that reference at instantiation and throw + * `DOMMatrix is not defined`, taking down every route/worker sharing the chunk. + * The lazy import (the exact pattern `local-extract.ts` documents) keeps pdfjs + + * `@napi-rs/canvas` (its Node canvas backend) out of the eager chunk. Rendering + * is pure local compute — no network, nothing written to disk. + * + * Bounds (denial-of-service + vision token cost): render at most the first + * `RASTER_MAX_PAGES` pages, each capped to `RASTER_TARGET_LONG_EDGE` on its long + * edge (never upscaled), JPEG at `RASTER_JPEG_QUALITY`. A discharge letter / lab + * is 1-4 pages; the cap keeps a pathological 200-page PDF from draining the + * subscription allowance in one job. + * + * NEVER throws: a malformed / encrypted / unrenderable PDF, a missing native + * binary, or any render error resolves to `{ ok: false }`, and the caller falls + * back to the local text-layer path or leaves the document un-indexed. This + * preserves the "a bad document never aborts an upload/batch" contract. + */ +import { Buffer } from "node:buffer"; + +import { annotate } from "@/lib/logging/context"; + +/** + * Whether the PDF rasterizer is available in this build. The native + * `@napi-rs/canvas` binary is compiled in and traced into the standalone image + * (see `next.config.ts`), so rasterization is a standing capability — every + * vision provider can read a PDF (Anthropic natively, all others via raster). + * The document capability DTO reads this so the UI offers a PDF read for a + * non-Anthropic provider; a runtime render failure still degrades gracefully + * (`prepareVisionInput` → `pdfNeedsAnthropic` → local text or un-indexed). + */ +export const RASTERIZATION_AVAILABLE = true; + +/** Render at most the first N pages — the token-cost / DoS governor. */ +export const RASTER_MAX_PAGES = 10; + +/** + * Longest-edge pixel cap per page. Tighter than the client upload downscale + * ceiling because these feed a token-metered vision model; ~1600px is legible + * for a document read without blowing up the tile count. + */ +export const RASTER_TARGET_LONG_EDGE = 1600; + +/** JPEG quality (0-100). ~80 keeps text legible at a fraction of PNG's bytes. */ +export const RASTER_JPEG_QUALITY = 80; + +/** One rasterized page, shaped as a vision `input_image` part. */ +export interface RasterImage { + mediaType: "image/jpeg"; + dataBase64: string; +} + +/** Best-effort rasterization outcome — never an exception. */ +export type RasterResult = { ok: true; images: RasterImage[] } | { ok: false }; + +// Minimal structural types for the slice of the pdfjs API this module uses. A +// type-only shape (erased at build) so we never eagerly evaluate the pdfjs +// module for its types; the real module is pulled at runtime via `import()`. +interface PdfViewport { + width: number; + height: number; +} +interface PdfCanvas { + width: number; + height: number; + toBuffer(mime: "image/jpeg", quality?: number): Buffer; +} +interface PdfCanvasAndContext { + canvas: PdfCanvas; + context: unknown; +} +interface PdfCanvasFactory { + create(width: number, height: number): PdfCanvasAndContext; + destroy?(target: PdfCanvasAndContext): void; +} +interface PdfPage { + getViewport(opts: { scale: number }): PdfViewport; + render(opts: { + canvasContext: unknown; + viewport: PdfViewport; + canvas: unknown; + }): { promise: Promise }; + cleanup(): void; +} +interface PdfDocument { + numPages: number; + canvasFactory: PdfCanvasFactory; + getPage(pageNumber: number): Promise; + destroy(): Promise; +} +interface PdfjsModule { + getDocument(opts: { + data: Uint8Array; + verbosity?: number; + isEvalSupported?: boolean; + useSystemFonts?: boolean; + }): { promise: Promise }; +} + +/** + * Render the first `RASTER_MAX_PAGES` pages of a PDF to bounded JPEG images. + * Returns `{ ok: false }` on any failure (malformed PDF, render throw, missing + * binary, or zero renderable pages) — the caller degrades to local text or + * un-indexed. Never throws. + */ +export async function rasterizePdf(buffer: Buffer): Promise { + let doc: PdfDocument | null = null; + try { + // Lazy import: keeps pdfjs (DOMMatrix) + @napi-rs/canvas out of the eager + // server chunk. Resolved at first rasterization, then module-cached. + const pdfjs = + (await import("pdfjs-dist/legacy/build/pdf.mjs")) as unknown as PdfjsModule; + + const task = pdfjs.getDocument({ + data: new Uint8Array(buffer), + verbosity: 0, // VerbosityLevel.ERRORS — no console spam on odd PDFs. + isEvalSupported: false, // never eval font programs (defence in depth). + useSystemFonts: true, + }); + doc = await task.promise; + + const pageCount = Math.min(doc.numPages, RASTER_MAX_PAGES); + const images: RasterImage[] = []; + + for (let pageNumber = 1; pageNumber <= pageCount; pageNumber++) { + const page = await doc.getPage(pageNumber); + try { + const base = page.getViewport({ scale: 1 }); + const longEdge = Math.max(base.width, base.height); + // Cap the long edge, never upscale (scale ≤ 1). + const scale = + longEdge > 0 ? Math.min(1, RASTER_TARGET_LONG_EDGE / longEdge) : 1; + const viewport = page.getViewport({ scale }); + + const factory = doc.canvasFactory; + const cc = factory.create( + Math.ceil(viewport.width), + Math.ceil(viewport.height), + ); + await page.render({ + canvasContext: cc.context, + viewport, + canvas: cc.canvas, + }).promise; + const jpeg = cc.canvas.toBuffer("image/jpeg", RASTER_JPEG_QUALITY); + images.push({ + mediaType: "image/jpeg", + dataBase64: jpeg.toString("base64"), + }); + factory.destroy?.(cc); + } finally { + page.cleanup(); + } + } + + if (images.length === 0) return { ok: false }; + annotate({ + action: { name: "documents.rasterize.ok" }, + meta: { pages: images.length, cappedFrom: doc.numPages }, + }); + return { ok: true, images }; + } catch (err) { + annotate({ + action: { name: "documents.rasterize.failed" }, + meta: { reason: err instanceof Error ? err.name : "unknown" }, + }); + return { ok: false }; + } finally { + if (doc) { + try { + await doc.destroy(); + } catch { + // Best-effort teardown; a destroy failure must not mask the result. + } + } + } +} diff --git a/src/lib/jobs/document-content-index-backfill.ts b/src/lib/jobs/document-content-index-backfill.ts index a9e78f413..fa5f17f05 100644 --- a/src/lib/jobs/document-content-index-backfill.ts +++ b/src/lib/jobs/document-content-index-backfill.ts @@ -137,7 +137,7 @@ export async function runContentIndexBackfillForUser( await reconcileSpend(userId, reservation.reserved, 0, dateKey); continue; } - const vision = prepareVisionInput(document, pick.pdfSupported); + const vision = await prepareVisionInput(document, pick.pdfSupported); if (!vision.ok) { // Not vision-indexable (should be filtered by MIME, but guard) — refund // and move the cursor past it so the walk keeps converging.