diff --git a/e2e/doctor-report.spec.ts b/e2e/doctor-report.spec.ts index 2ce774311..764b658c3 100644 --- a/e2e/doctor-report.spec.ts +++ b/e2e/doctor-report.spec.ts @@ -49,7 +49,8 @@ import { readFile } from "node:fs/promises"; import type { Page, Response } from "@playwright/test"; -import { PDFParse } from "pdf-parse"; + +import { readPdfText } from "@/lib/documents/pdfjs"; import { E2E_REPORT_OWNER, @@ -342,13 +343,7 @@ async function generateReport(page: Page): Promise { expect(response.headers()["content-type"] ?? "").toContain("application/pdf"); expect(bytes.byteLength).toBeGreaterThan(MIN_PDF_BYTES); - const parser = new PDFParse({ data: new Uint8Array(bytes) }); - let text: string; - try { - text = (await parser.getText()).text; - } finally { - await parser.destroy(); - } + const text = await readPdfText(new Uint8Array(bytes)); return { status: response.status(), diff --git a/src/app/api/export/health-record/__tests__/route.test.ts b/src/app/api/export/health-record/__tests__/route.test.ts index d60b0c635..8e797ae7e 100644 --- a/src/app/api/export/health-record/__tests__/route.test.ts +++ b/src/app/api/export/health-record/__tests__/route.test.ts @@ -14,9 +14,10 @@ */ import { describe, it, expect, vi, beforeEach } from "vitest"; import { NextRequest } from "next/server"; -import { PDFParse } from "pdf-parse"; import { unzipSync } from "fflate"; +import { readPdfText } from "@/lib/documents/pdfjs"; + vi.mock("@/lib/db", () => ({ prisma: { // `groupBy` sizes the report's dense windows (#1023): a sparse account. @@ -155,22 +156,11 @@ function mkReq( } async function pdfText(res: Response): Promise { - const bytes = new Uint8Array(await res.arrayBuffer()); - const parser = new PDFParse({ data: bytes }); - try { - return (await parser.getText()).text; - } finally { - await parser.destroy(); - } + return readPdfText(new Uint8Array(await res.arrayBuffer())); } -async function pdfBytesText(bytes: Uint8Array): Promise { - const parser = new PDFParse({ data: bytes }); - try { - return (await parser.getText()).text; - } finally { - await parser.destroy(); - } +function pdfBytesText(bytes: Uint8Array): Promise { + return readPdfText(bytes); } beforeEach(() => { diff --git a/src/lib/__tests__/doctor-report-pdf-core.test.ts b/src/lib/__tests__/doctor-report-pdf-core.test.ts index 70e15692c..0adcc830c 100644 --- a/src/lib/__tests__/doctor-report-pdf-core.test.ts +++ b/src/lib/__tests__/doctor-report-pdf-core.test.ts @@ -1,5 +1,4 @@ import { describe, it, expect } from "vitest"; -import { PDFParse } from "pdf-parse"; import { buildDoctorReportPdfDocument, renderDoctorReportPdfBytes, @@ -12,15 +11,10 @@ import type { DoctorReportData } from "../doctor-report-data"; import { computeGlucoseClinicalMetrics } from "@/lib/analytics/glucose-metrics"; import { getServerTranslator } from "../i18n/server-translator"; import { reportMeasurementLabelKey } from "../report-selection/catalogue"; +import { readPdfText } from "@/lib/documents/pdfjs"; -async function extractText(bytes: Uint8Array): Promise { - const parser = new PDFParse({ data: bytes }); - try { - const result = await parser.getText(); - return result.text; - } finally { - await parser.destroy(); - } +function extractText(bytes: Uint8Array): Promise { + return readPdfText(bytes); } const FIXED_NOW = new Date("2026-05-03T12:00:00.000Z"); diff --git a/src/lib/doctor-report-pdf/__tests__/activity-effect-sections.test.ts b/src/lib/doctor-report-pdf/__tests__/activity-effect-sections.test.ts index 46aa08183..85587fa4d 100644 --- a/src/lib/doctor-report-pdf/__tests__/activity-effect-sections.test.ts +++ b/src/lib/doctor-report-pdf/__tests__/activity-effect-sections.test.ts @@ -5,8 +5,8 @@ * block. Text is read back out of the real PDF bytes. */ import { describe, expect, it } from "vitest"; -import { PDFParse } from "pdf-parse"; +import { readPdfText } from "@/lib/documents/pdfjs"; import { renderDoctorReportPdfBytes } from "@/lib/doctor-report-pdf-core"; import type { DoctorReportData } from "@/lib/doctor-report-data"; import { computeGlucoseClinicalMetrics } from "@/lib/analytics/glucose-metrics"; @@ -16,13 +16,8 @@ import type { EfficacySegmented } from "@/lib/medications/efficacy/segmented-eff const NOW = new Date("2026-05-03T12:00:00.000Z"); -async function textOf(bytes: Uint8Array): Promise { - const parser = new PDFParse({ data: bytes }); - try { - return (await parser.getText()).text; - } finally { - await parser.destroy(); - } +function textOf(bytes: Uint8Array): Promise { + return readPdfText(bytes); } function series(values: number[]) { diff --git a/src/lib/documents/__tests__/local-extract.test.ts b/src/lib/documents/__tests__/local-extract.test.ts index fc641ea34..21640fc5f 100644 --- a/src/lib/documents/__tests__/local-extract.test.ts +++ b/src/lib/documents/__tests__/local-extract.test.ts @@ -8,7 +8,12 @@ import { jsPDF } from "jspdf"; * (unsupported); malformed bytes fail closed to "error" without throwing. */ -vi.mock("@/lib/logging/context", () => ({ annotate: vi.fn() })); +const { annotate, logCaught } = vi.hoisted(() => ({ + annotate: vi.fn(), + logCaught: vi.fn(), +})); +vi.mock("@/lib/logging/context", () => ({ annotate })); +vi.mock("@/lib/logging/signal", () => ({ logCaught })); import { extractPdfText, @@ -55,6 +60,52 @@ describe("extractPdfText", () => { const result = await extractPdfText(Buffer.from([1, 2, 3, 4, 5, 6, 7, 8])); expect(result).toEqual({ ok: false, reason: "error" }); }); + + // #1277: a failed read used to leave nothing behind but `error`, which + // the index job reports under another name. It now says what pdf.js threw. + it("says why a read failed", async () => { + annotate.mockClear(); + logCaught.mockClear(); + const bytes = Buffer.from([1, 2, 3, 4, 5, 6, 7, 8]); + await extractPdfText(bytes); + expect(logCaught).toHaveBeenCalledOnce(); + const [action, error, meta] = logCaught.mock.calls[0]!; + expect(action).toBe("documents.local_extract.failed"); + expect(error).toBeInstanceOf(Error); + expect(meta).toEqual({ bytes: 8 }); + expect(annotate).toHaveBeenCalledWith({ + action: { name: "documents.localIndex.extractFailed" }, + meta: { localExtractError: (error as Error).name }, + }); + }); + + it("reads a PDF with an owner password only, and refuses one that needs a password to open", async () => { + const owned = new jsPDF({ + encryption: { ownerPassword: "owner", userPermissions: ["print"] }, + }); + owned.text("Encrypted lab report haemoglobin creatinine glucose", 10, 10); + const readable = await extractPdfText( + Buffer.from(owned.output("arraybuffer")), + ); + expect(readable).toMatchObject({ ok: true, source: "local-pdf" }); + + logCaught.mockClear(); + const locked = new jsPDF({ + encryption: { + userPassword: "user", + ownerPassword: "owner", + userPermissions: ["print"], + }, + }); + locked.text("Encrypted lab report haemoglobin creatinine glucose", 10, 10); + const result = await extractPdfText( + Buffer.from(locked.output("arraybuffer")), + ); + expect(result).toEqual({ ok: false, reason: "error" }); + expect((logCaught.mock.calls[0]![1] as Error).name).toBe( + "PasswordException", + ); + }); }); describe("localExtractText", () => { diff --git a/src/lib/documents/__tests__/pdfjs-one-runtime-raster-first.test.ts b/src/lib/documents/__tests__/pdfjs-one-runtime-raster-first.test.ts new file mode 100644 index 000000000..1af8dc44f --- /dev/null +++ b/src/lib/documents/__tests__/pdfjs-one-runtime-raster-first.test.ts @@ -0,0 +1,26 @@ +import { describe, it, vi } from "vitest"; + +/** + * GitHub #1277, rasterization first: a render used to claim the in-process + * pdf.js worker for its own copy, and every later local text read failed + * without a word, leaving the document out of the search index. See + * `pdfjs-one-runtime.ts`. + */ + +vi.mock("@/lib/logging/context", () => ({ annotate: vi.fn() })); +vi.mock("@/lib/logging/signal", () => ({ logCaught: vi.fn() })); + +import { + expectFreshProcess, + expectRaster, + expectTextRead, +} from "./pdfjs-one-runtime"; + +describe("one pdf.js per process (#1277): rasterization first", () => { + it("reads text after a rasterization, and rasterizes again after that", async () => { + expectFreshProcess(); + await expectRaster(); + await expectTextRead(); + await expectRaster(); + }); +}); diff --git a/src/lib/documents/__tests__/pdfjs-one-runtime-text-first.test.ts b/src/lib/documents/__tests__/pdfjs-one-runtime-text-first.test.ts new file mode 100644 index 000000000..846ec1852 --- /dev/null +++ b/src/lib/documents/__tests__/pdfjs-one-runtime-text-first.test.ts @@ -0,0 +1,26 @@ +import { describe, it, vi } from "vitest"; + +/** + * GitHub #1277, text first: a local text read used to claim the in-process + * pdf.js worker for its own copy, and every later rasterization (thumbnails, + * the vision read of a scan) failed until a restart. See + * `pdfjs-one-runtime.ts`. + */ + +vi.mock("@/lib/logging/context", () => ({ annotate: vi.fn() })); +vi.mock("@/lib/logging/signal", () => ({ logCaught: vi.fn() })); + +import { + expectFreshProcess, + expectRaster, + expectTextRead, +} from "./pdfjs-one-runtime"; + +describe("one pdf.js per process (#1277): text read first", () => { + it("rasterizes after a text read, and reads text again after that", async () => { + expectFreshProcess(); + await expectTextRead(); + await expectRaster(); + await expectTextRead(); + }); +}); diff --git a/src/lib/documents/__tests__/pdfjs-one-runtime.ts b/src/lib/documents/__tests__/pdfjs-one-runtime.ts new file mode 100644 index 000000000..d3fb7a672 --- /dev/null +++ b/src/lib/documents/__tests__/pdfjs-one-runtime.ts @@ -0,0 +1,50 @@ +/** + * Shared steps for the two `pdfjs-one-runtime-*` files (GitHub #1277). + * + * Under Node, pdf.js runs its worker in-process and keeps it on + * `globalThis.pdfjsWorker`. Whichever pdf.js copy opens a document first + * owns that global for the life of the process, and a different copy then + * refuses every document with "The API version … does not match the Worker + * version …". The text reader and the rasterizer once loaded different + * copies, so whichever ran first broke the other until a restart. + * + * Each order needs a fresh process, because the global cannot be taken back + * once claimed; that is why the two orders live in two files (the isolated + * project gives each file its own process) instead of two tests here. + */ +import { readFileSync } from "node:fs"; +import { join } from "node:path"; + +import { expect } from "vitest"; + +import { extractPdfText } from "../local-extract"; +import { rasterizePdf } from "../rasterize-pdf"; + +const FIXTURE = join( + process.cwd(), + "tests", + "fixtures", + "pdf", + "fallback-fonts.pdf", +); + +const pdf = () => readFileSync(FIXTURE); + +/** Nothing in this process has opened a PDF yet. */ +export function expectFreshProcess(): void { + expect( + (globalThis as { pdfjsWorker?: unknown }).pdfjsWorker, + "the file must run in its own process; another file already opened a PDF", + ).toBeUndefined(); +} + +export async function expectTextRead(): Promise { + const result = await extractPdfText(pdf()); + expect(result).toMatchObject({ ok: true, source: "local-pdf" }); + if (result.ok) expect(result.text).toContain("Test Insurance AG"); +} + +export async function expectRaster(): Promise { + const result = await rasterizePdf(pdf(), 1); + expect(result).toMatchObject({ ok: true }); +} diff --git a/src/lib/documents/__tests__/pdfjs-text-layer.test.ts b/src/lib/documents/__tests__/pdfjs-text-layer.test.ts new file mode 100644 index 000000000..aae80e893 --- /dev/null +++ b/src/lib/documents/__tests__/pdfjs-text-layer.test.ts @@ -0,0 +1,108 @@ +import { jsPDF } from "jspdf"; +import { describe, expect, it } from "vitest"; + +import { pageTextLayer, readPdfText, type PdfTextItem } from "../pdfjs"; + +/** + * The text layer as the content index reads it. The layout is the one the + * local text read produced before it moved onto the app's own pdf.js (#1277): + * a break where pdf.js marks one or the next item sits on a lower line, a tab + * between far-apart items on one line, each page followed by a blank line. + */ + +/** A page whose viewport maps PDF space to itself, flipped like pdf.js. */ +function pageOf(items: PdfTextItem[]) { + return { + getViewport: () => ({ + width: 600, + height: 800, + convertToViewportPoint: (x: number, y: number) => [x, 800 - y], + }), + getTextContent: async () => ({ items }), + }; +} + +function item( + str: string, + x: number, + y: number, + extra: Partial = {}, +): PdfTextItem { + return { + str, + transform: [10, 0, 0, 10, x, y], + width: str.length * 5, + height: 10, + hasEOL: false, + ...extra, + }; +} + +describe("pageTextLayer", () => { + it("breaks once where pdf.js marks the end of a line", async () => { + const text = await pageTextLayer( + pageOf([ + item("Haemoglobin", 10, 700, { hasEOL: true }), + item("14.2 g/dL", 10, 688), + ]), + ); + expect(text).toBe("Haemoglobin\n14.2 g/dL"); + }); + + it("breaks when the next item sits a line lower without a marked end", async () => { + const text = await pageTextLayer( + pageOf([item("Creatinine", 10, 700), item("0.9 mg/dL", 10, 680)]), + ); + expect(text).toBe("Creatinine\n0.9 mg/dL"); + }); + + it("does not break for a small baseline shift on one line", async () => { + const text = await pageTextLayer( + pageOf([item("10", 10, 700), item("3", 21, 703)]), + ); + expect(text).toBe("103"); + }); + + it("separates far-apart items on one line with a tab", async () => { + const text = await pageTextLayer( + pageOf([item("Glucose", 10, 700), item("92", 200, 700)]), + ); + expect(text).toBe("Glucose\t92"); + }); + + it("skips marked-content items, which carry no text", async () => { + const text = await pageTextLayer( + pageOf([{ hasEOL: false } as PdfTextItem, item("Text", 10, 700)]), + ); + expect(text).toBe("Text"); + }); +}); + +describe("readPdfText", () => { + function pages(count: number): Buffer { + const doc = new jsPDF(); + for (let p = 1; p <= count; p++) { + if (p > 1) doc.addPage(); + doc.text(`Page ${p} lab report`, 10, 10); + doc.text("second line", 10, 20); + } + return Buffer.from(doc.output("arraybuffer")); + } + + it("joins pages with a blank line and adds no page markers", async () => { + const text = await readPdfText(pages(2)); + expect(text).toBe( + "Page 1 lab report\nsecond line\n\nPage 2 lab report\nsecond line\n\n", + ); + }); + + it("reads no further than the page cap", async () => { + const text = await readPdfText(pages(3), 2); + expect(text).toContain("Page 2"); + expect(text).not.toContain("Page 3"); + }); + + it("throws what pdf.js throws on a file that is not a PDF", async () => { + await expect(readPdfText(Buffer.from("not a pdf"))).rejects.toThrow(); + }); +}); diff --git a/src/lib/documents/__tests__/rasterize-pdf-font-data.test.ts b/src/lib/documents/__tests__/rasterize-pdf-font-data.test.ts index 86fd7c5fe..298ccf4dd 100644 --- a/src/lib/documents/__tests__/rasterize-pdf-font-data.test.ts +++ b/src/lib/documents/__tests__/rasterize-pdf-font-data.test.ts @@ -61,6 +61,7 @@ vi.mock("@napi-rs/canvas", async (importActual) => { }; }); +import { __resetPdfjsAssetsForTests } from "../pdfjs"; import * as raster from "../rasterize-pdf"; import { generateThumbnail } from "../thumbnail"; @@ -85,7 +86,8 @@ beforeEach(() => { annotate.mockClear(); registerFromPath.mockClear(); getDocumentOptions.length = 0; - raster.__resetPdfjsAssetsForTests(); + __resetPdfjsAssetsForTests(); + raster.__resetFallbackFamiliesForTests(); }); describe("rasterizePdf font and decoder data (#1124)", () => { diff --git a/src/lib/documents/content-index.ts b/src/lib/documents/content-index.ts index 4b88616f3..8690260d7 100644 --- a/src/lib/documents/content-index.ts +++ b/src/lib/documents/content-index.ts @@ -325,7 +325,7 @@ export async function loadDocumentChatText( * - `vision` → an AI provider transcribed the stored original (image or PDF, * incl. scanned) — the AI-first primary path. * - `text-ocr` → browser-OCR text posted by the client (opt-in local OCR). - * - `local-pdf` → server-side text-layer extraction of a PDF (`pdf-parse`), no + * - `local-pdf` → server-side text-layer extraction of a PDF (pdf.js), no * provider, no egress — the fallback when no provider is usable. * - `local-ocr` → server-side OCR of a scanned image/PDF (deferred follow-up). */ diff --git a/src/lib/documents/index-document.ts b/src/lib/documents/index-document.ts index 80c3ff25d..60afab046 100644 --- a/src/lib/documents/index-document.ts +++ b/src/lib/documents/index-document.ts @@ -24,7 +24,7 @@ * configured, the toggle is OFF for an external pick, the provider can't * read this file (e.g. a text-layer PDF on a non-Anthropic account), the * daily budget is reached, or the provider call fails — fall back to - * server-side text-layer extraction (`pdf-parse`, no egress, milliseconds). + * server-side text-layer extraction (pdf.js, no egress, milliseconds). * So content search works at a baseline even with no AI, and every * text-layer PDF is searchable regardless of provider. * diff --git a/src/lib/documents/local-extract.ts b/src/lib/documents/local-extract.ts index d473bd043..b41598bc6 100644 --- a/src/lib/documents/local-extract.ts +++ b/src/lib/documents/local-extract.ts @@ -3,9 +3,9 @@ * * When NO AI provider is configured — or the user has not consented to one — * content search still needs a baseline. This module extracts a PDF's embedded - * text layer SERVER-SIDE with `pdf-parse` (already a direct dependency; its - * `getText()` touches only the pure-JS/wasm pdfjs core, never the native - * `@napi-rs/canvas`), in milliseconds, with no OCR and no third-party egress. + * text layer SERVER-SIDE through the app's one pdf.js (`./pdfjs`; reading the + * text layer never draws, so the native canvas does no work here), in + * milliseconds, with no OCR and no third-party egress. * The extracted text feeds the SAME blind, encrypted content index the provider * path writes (`upsertContentIndex`), so nothing readable is stored at rest. * @@ -20,18 +20,10 @@ */ import { Buffer } from "node:buffer"; -// `pdf-parse` pulls in pdfjs, whose module top-level references the browser-only -// `DOMMatrix` global. A STATIC import makes the Turbopack server chunk evaluate -// that reference at instantiation and throw `DOMMatrix is not defined`, taking -// down every route/worker that shares the chunk (the document routes among -// them). The value is therefore loaded lazily inside `extractPdfText` via a -// runtime `import()`, which lets pdf-parse initialise pdfjs on its own terms and -// never touches DOMMatrix at load — verified to extract a real text layer under -// plain Node. Keep the surface type as a type-only import (fully erased at -// build, no runtime module evaluation). -import type { PDFParse } from "pdf-parse"; - import { annotate } from "@/lib/logging/context"; +import { logCaught } from "@/lib/logging/signal"; + +import { readPdfText } from "./pdfjs"; /** * Minimum joined text length for a PDF to count as a real text-layer document. @@ -63,39 +55,34 @@ export type LocalExtractResult = * Extract a text-layer PDF's embedded text server-side. Returns `empty` for a * scanned/no-text-layer PDF and `error` for a malformed/encrypted one — never * throws, so a bad document can never abort the caller (the upload/backfill job - * just leaves it un-indexed). + * just leaves it un-indexed). An `error` leaves a `warn` line naming what + * pdf.js threw, because the outcome alone does not say why a document stayed + * out of the index. */ export async function extractPdfText( buffer: Buffer, ): Promise { - let parser: PDFParse | null = null; + let raw: string; try { - // Lazy load: keeps pdfjs/DOMMatrix out of the eager server chunk (see the - // import-site note). Resolved at first PDF extraction, then module-cached. - const { PDFParse } = await import("pdf-parse"); - parser = new PDFParse({ data: new Uint8Array(buffer) }); - const result = await parser.getText({ - // Bound the page walk and drop the default "-- N of M --" page markers so - // only real document text is tokenised. - first: LOCAL_MAX_PAGES, - pageJoiner: "", + // Bounded page walk; pages joined by a blank line, with no page markers, + // so only real document text is tokenised. + raw = await readPdfText(buffer, LOCAL_MAX_PAGES); + } catch (err) { + const errorType = err instanceof Error ? err.name || "Error" : typeof err; + annotate({ + action: { name: "documents.localIndex.extractFailed" }, + meta: { localExtractError: errorType }, + }); + logCaught("documents.local_extract.failed", err, { + bytes: buffer.length, }); - const text = (result.text ?? "").trim(); - if (text.length < LOCAL_TEXT_MIN_CHARS) { - return { ok: false, reason: "empty" }; - } - return { ok: true, text, source: "local-pdf" }; - } catch { return { ok: false, reason: "error" }; - } finally { - if (parser) { - try { - await parser.destroy(); - } catch { - // Best-effort teardown; a destroy failure must not mask the result. - } - } } + const text = raw.trim(); + if (text.length < LOCAL_TEXT_MIN_CHARS) { + return { ok: false, reason: "empty" }; + } + return { ok: true, text, source: "local-pdf" }; } /** diff --git a/src/lib/documents/pdfjs.ts b/src/lib/documents/pdfjs.ts new file mode 100644 index 000000000..5d44c1f31 --- /dev/null +++ b/src/lib/documents/pdfjs.ts @@ -0,0 +1,288 @@ +/** + * The one pdf.js this server runs, and the one way it is opened. + * + * Under Node, pdf.js does not start a worker thread: it loads its worker code + * into the same process and keeps it on `globalThis.pdfjsWorker`. Whichever + * copy of pdf.js opens a document first owns that global until the process + * exits, and any other copy then refuses every document with "The API version + * … does not match the Worker version …". The server used to load two copies, + * one for the local text read and one for rasterizing, so whichever ran first + * broke the other until a restart (#1277). Both paths now open documents + * here, through the app's own `pdfjs-dist`, with the same options; + * `src/__tests__/single-pdfjs-guard.test.ts` keeps a second copy out. + * + * pdf.js is loaded lazily with a runtime `import()`: its module top level + * reads the browser-only `DOMMatrix` global, so a static import would make the + * server chunk evaluate it at instantiation and throw `DOMMatrix is not + * defined`, taking down every route and worker sharing the chunk. The import + * resolves once and is module-cached after that. + * + * This module has no logging of its own, so the end-to-end suite can read + * report text with it; the callers say what failed. + */ +import { Buffer } from "node:buffer"; +import { existsSync } from "node:fs"; +import { createRequire } from "node:module"; +import { dirname, join } from "node:path"; + +// Structural types for the slice of the pdf.js API used here and in the +// rasterizer. Type-only, so nothing evaluates pdf.js for its types. +export interface PdfViewport { + width: number; + height: number; + convertToViewportPoint(x: number, y: number): number[]; +} +export interface PdfCanvas { + width: number; + height: number; + toBuffer(mime: "image/jpeg", quality?: number): Buffer; +} +export interface PdfContext2D { + getImageData( + x: number, + y: number, + width: number, + height: number, + ): { data: Uint8ClampedArray }; +} +interface PdfCanvasAndContext { + canvas: PdfCanvas; + context: PdfContext2D; +} +interface PdfCanvasFactory { + create(width: number, height: number): PdfCanvasAndContext; + destroy?(target: PdfCanvasAndContext): void; +} +/** A text-layer item; marked-content items carry no `str` and are skipped. */ +export interface PdfTextItem { + str?: string; + hasEOL?: boolean; + transform?: number[]; + width?: number; + height?: number; +} +export interface PdfPage { + getViewport(opts: { scale: number }): PdfViewport; + render(opts: { + canvasContext: unknown; + viewport: PdfViewport; + canvas: unknown; + }): { promise: Promise }; + cleanup(): void; + getTextContent(): Promise<{ items: PdfTextItem[] }>; +} +export interface PdfDocument { + numPages: number; + canvasFactory: PdfCanvasFactory; + getPage(pageNumber: number): Promise; +} +export interface PdfLoadingTask { + promise: Promise; + destroy(): Promise; +} +interface PdfDocumentOptions { + data: Uint8Array; + verbosity?: number; + isEvalSupported?: boolean; + useSystemFonts?: boolean; + standardFontDataUrl?: string; + cMapUrl?: string; + cMapPacked?: boolean; + wasmUrl?: string; +} +interface PdfjsModule { + getDocument(opts: PdfDocumentOptions): PdfLoadingTask; +} + +/** + * Where pdf.js finds the data it does not carry in its JS: the standard 14 + * fonts (a PDF that names Helvetica or Times without embedding them), the + * CMaps (CID fonts with a predefined encoding), and the wasm image decoders + * (JPEG 2000, JBIG2). In a browser pdf.js fetches these from a URL; under + * Node they are read from disk, and without a directory pdf.js draws the text + * without glyphs and cannot map a predefined-CMap font to characters. + * + * The runtime image carries no system fonts at all, so nothing else catches + * that: a letter set in a non-embedded font rendered as an empty page in the + * container while it rendered correctly on a workstation, where the operating + * system's fonts stood in. + * + * Resolved from the working directory with Node's own resolution, which is + * what finds the hoisted `node_modules/pdfjs-dist` in the standalone image + * and the pnpm link in a checkout. The specifier is assembled at runtime so + * the bundler leaves the lookup alone. The ICC profile option is not set: its + * loader needs a synchronous XMLHttpRequest that Node lacks, and pdf.js falls + * back to each colour space's alternate. + */ +export interface PdfjsAssets { + standardFontDataUrl: string; + cMapUrl: string; + wasmUrl: string; + /** The Liberation Sans faces the rasterizer registers as fallback families. */ + fallbackFonts: string[]; +} + +let cachedAssets: PdfjsAssets | null | undefined; + +export function resolvePdfjsAssets(): PdfjsAssets | null { + if (cachedAssets !== undefined) return cachedAssets; + cachedAssets = null; + try { + const anchor = createRequire(join(process.cwd(), "package.json")); + const root = dirname( + anchor.resolve(["pdfjs-dist", "package.json"].join("/")), + ); + const fonts = join(root, "standard_fonts"); + const assets: PdfjsAssets = { + standardFontDataUrl: `${fonts}/`, + cMapUrl: `${join(root, "cmaps")}/`, + wasmUrl: `${join(root, "wasm")}/`, + fallbackFonts: [ + "LiberationSans-Regular.ttf", + "LiberationSans-Bold.ttf", + "LiberationSans-Italic.ttf", + "LiberationSans-BoldItalic.ttf", + ].map((file) => join(fonts, file)), + }; + if ( + existsSync(join(fonts, "FoxitSerif.pfb")) && + existsSync(join(root, "cmaps", "UniJIS-UTF16-H.bcmap")) && + existsSync(join(root, "wasm", "openjpeg.wasm")) && + assets.fallbackFonts.every((file) => existsSync(file)) + ) { + cachedAssets = assets; + } + } catch { + // Unresolvable: open without the data; the caller says so. + } + return cachedAssets; +} + +/** Test hook: the asset lookup is per process. */ +export function __resetPdfjsAssetsForTests(): void { + cachedAssets = undefined; +} + +/** + * Start loading a PDF. The caller awaits `task.promise` and always awaits + * `task.destroy()` afterwards, which also tears the document down. + */ +export async function openPdfDocument( + data: Uint8Array | Buffer, +): Promise { + const pdfjs = + (await import("pdfjs-dist/legacy/build/pdf.mjs")) as unknown as PdfjsModule; + const assets = resolvePdfjsAssets(); + return pdfjs.getDocument({ + // A copy: pdf.js transfers the buffer it is given to its worker. + data: new Uint8Array(data), + verbosity: 0, // VerbosityLevel.ERRORS: no console noise on odd PDFs. + isEvalSupported: false, // never eval font programs (defence in depth). + // There are no system fonts to use: substitutes come from pdf.js' own + // standard font data (and, when rendering, the registered fallbacks). + useSystemFonts: false, + ...(assets + ? { + standardFontDataUrl: assets.standardFontDataUrl, + cMapUrl: assets.cMapUrl, + cMapPacked: true, + wasmUrl: assets.wasmUrl, + } + : {}), + }); +} + +/** Vertical gap (pt) past which the next item starts a new line. */ +const LINE_THRESHOLD = 4.6; +/** Horizontal gap (pt) on one line past which the next item is a new cell. */ +const CELL_THRESHOLD = 7; + +/** + * One page's text layer as plain text: items in content order, a line break + * where pdf.js marks one or where the next item sits on a lower line, and a + * tab between items on one line that are far apart (table cells). + * + * This is the layout `pdf-parse` produced with its defaults, kept so that + * swapping it out did not change what the content index sees. + */ +export async function pageTextLayer( + page: Pick, +): Promise { + const viewport = page.getViewport({ scale: 1 }); + const { items } = await page.getTextContent(); + const parts: string[] = []; + let lastX: number | undefined; + let lastY: number | undefined; + let lineHeight = 0; + for (const item of items) { + if (typeof item.str !== "string") continue; + const transform = item.transform ?? [1, 0, 0, 1, 0, 0]; + const [x = 0, y = 0] = viewport.convertToViewportPoint( + transform[4] ?? 0, + transform[5] ?? 0, + ); + let str = item.str; + if (lastY !== undefined) { + const gap = Math.abs(lastY - y); + if (gap > LINE_THRESHOLD) { + const previous = parts.at(-1); + const opensLine = + str.startsWith("\n") || (str.trim() === "" && item.hasEOL === true); + if ( + previous !== undefined && + !previous.endsWith("\n") && + !opensLine && + gap - 1 > lineHeight + ) { + parts.push("\n"); + lineHeight = 0; + } + } else if ( + gap < LINE_THRESHOLD && + lastX !== undefined && + Math.abs(lastX - x) > CELL_THRESHOLD + ) { + str = `\t${str}`; + } + } + parts.push(str); + lastX = x + (item.width ?? 0); + lastY = y; + lineHeight = Math.max(lineHeight, item.height ?? 0); + if (item.hasEOL) parts.push("\n"); + if (item.hasEOL || str.endsWith("\n")) lineHeight = 0; + } + return parts.join(""); +} + +/** + * The text layer of the first `maxPages` pages, each page followed by a blank + * line. Throws what pdf.js throws (a malformed file, a password, a page that + * will not parse); the document is torn down either way. + */ +export async function readPdfText( + data: Uint8Array | Buffer, + maxPages = Number.POSITIVE_INFINITY, +): Promise { + const task = await openPdfDocument(data); + try { + const doc = await task.promise; + const last = Math.min(doc.numPages, maxPages); + let text = ""; + for (let pageNumber = 1; pageNumber <= last; pageNumber++) { + const page = await doc.getPage(pageNumber); + try { + text += `${await pageTextLayer(page)}\n\n`; + } finally { + page.cleanup(); + } + } + return text; + } finally { + try { + await task.destroy(); + } catch { + // Best-effort teardown; a destroy failure must not mask the result. + } + } +} diff --git a/src/lib/documents/rasterize-pdf.ts b/src/lib/documents/rasterize-pdf.ts index c7345ecae..f23730e14 100644 --- a/src/lib/documents/rasterize-pdf.ts +++ b/src/lib/documents/rasterize-pdf.ts @@ -11,13 +11,11 @@ * higher-fidelity native PDF block; a text-layer PDF is still read locally for * free before we ever rasterize). * - * pdfjs is loaded LAZILY via a runtime `import()` — its module top-level - * references the browser-only `DOMMatrix` global, so a STATIC import would make - * the Turbopack server chunk evaluate that reference at instantiation and throw - * `DOMMatrix is not defined`, taking down every route/worker sharing the chunk. - * The lazy import (the exact pattern `local-extract.ts` documents) keeps pdfjs + - * `@napi-rs/canvas` (its Node canvas backend) out of the eager chunk. Rendering - * is pure local compute — no network, nothing written to disk. + * pdf.js is opened through `./pdfjs`, the one loader the local text read uses + * too (two copies in one process break each other, #1277). It loads pdf.js + * and `@napi-rs/canvas` (its Node canvas backend) lazily, keeping both out of + * the eager server chunk. Rendering is pure local compute — no network, + * nothing written to disk. * * Bounds (denial-of-service + vision token cost): render at most the first * `RASTER_MAX_PAGES` pages, each capped to `RASTER_TARGET_LONG_EDGE` on its long @@ -31,13 +29,19 @@ * preserves the "a bad document never aborts an upload/batch" contract. */ import { Buffer } from "node:buffer"; -import { existsSync } from "node:fs"; -import { createRequire } from "node:module"; -import { dirname, join } from "node:path"; import { annotate } from "@/lib/logging/context"; import { nativeCanvasSupported } from "./native-canvas-support"; +import { + openPdfDocument, + resolvePdfjsAssets, + type PdfContext2D, + type PdfDocument, + type PdfjsAssets, + type PdfLoadingTask, + type PdfPage, +} from "./pdfjs"; /** * Whether the PDF rasterizer is available in this build. The native @@ -110,137 +114,10 @@ export type RasterResult = } | { ok: false; reason: "unsupported" | "render-failed" }; -// Minimal structural types for the slice of the pdfjs API this module uses. A -// type-only shape (erased at build) so we never eagerly evaluate the pdfjs -// module for its types; the real module is pulled at runtime via `import()`. -interface PdfViewport { - width: number; - height: number; -} -interface PdfCanvas { - width: number; - height: number; - toBuffer(mime: "image/jpeg", quality?: number): Buffer; -} -interface PdfContext2D { - getImageData( - x: number, - y: number, - width: number, - height: number, - ): { data: Uint8ClampedArray }; -} -interface PdfCanvasAndContext { - canvas: PdfCanvas; - context: PdfContext2D; -} -interface PdfCanvasFactory { - create(width: number, height: number): PdfCanvasAndContext; - destroy?(target: PdfCanvasAndContext): void; -} -interface PdfPage { - getViewport(opts: { scale: number }): PdfViewport; - render(opts: { - canvasContext: unknown; - viewport: PdfViewport; - canvas: unknown; - }): { promise: Promise }; - cleanup(): void; - getTextContent(): Promise<{ - items: Array<{ str?: string; hasEOL?: boolean }>; - }>; -} -interface PdfDocument { - numPages: number; - canvasFactory: PdfCanvasFactory; - getPage(pageNumber: number): Promise; -} -interface PdfLoadingTask { - promise: Promise; - destroy(): Promise; -} -interface PdfjsModule { - getDocument(opts: PdfDocumentOptions): PdfLoadingTask; -} -interface PdfDocumentOptions { - data: Uint8Array; - verbosity?: number; - isEvalSupported?: boolean; - useSystemFonts?: boolean; - standardFontDataUrl?: string; - cMapUrl?: string; - cMapPacked?: boolean; - wasmUrl?: string; -} interface GlobalFontsLike { registerFromPath(path: string, nameAlias?: string): unknown; } -/** - * Where pdfjs finds the data it does not carry in its JS: the standard 14 - * fonts (a PDF that names Helvetica or Times without embedding them), the - * CMaps (CID fonts with a predefined encoding), and the wasm image decoders - * (JPEG 2000, JBIG2). In a browser pdfjs fetches these from a URL; under Node - * they are read from disk, and without a directory pdfjs simply draws the text - * without glyphs. - * - * The runtime image carries no system fonts at all, so nothing else catches - * that: a letter set in a non-embedded font rendered as an empty page in the - * container while it rendered correctly on a workstation, where the operating - * system's fonts stood in. The image-only provider then read the empty page - * and returned a line of nothing. - * - * Resolved from the working directory with Node's own resolution, which is - * what finds the hoisted `node_modules/pdfjs-dist` in the standalone image - * and the pnpm link in a checkout. The specifier is assembled at runtime so - * the bundler leaves the lookup alone. The ICC profile option is not set: its - * loader needs a synchronous XMLHttpRequest that Node lacks, and pdfjs falls - * back to each colour space's alternate. - */ -export interface PdfjsAssets { - standardFontDataUrl: string; - cMapUrl: string; - wasmUrl: string; - /** The Liberation Sans faces registered as the generic fallback families. */ - fallbackFonts: string[]; -} - -let cachedAssets: PdfjsAssets | null | undefined; - -export function resolvePdfjsAssets(): PdfjsAssets | null { - if (cachedAssets !== undefined) return cachedAssets; - cachedAssets = null; - try { - const anchor = createRequire(join(process.cwd(), "package.json")); - const root = dirname( - anchor.resolve(["pdfjs-dist", "package.json"].join("/")), - ); - const fonts = join(root, "standard_fonts"); - const assets: PdfjsAssets = { - standardFontDataUrl: `${fonts}/`, - cMapUrl: `${join(root, "cmaps")}/`, - wasmUrl: `${join(root, "wasm")}/`, - fallbackFonts: [ - "LiberationSans-Regular.ttf", - "LiberationSans-Bold.ttf", - "LiberationSans-Italic.ttf", - "LiberationSans-BoldItalic.ttf", - ].map((file) => join(fonts, file)), - }; - if ( - existsSync(join(fonts, "FoxitSerif.pfb")) && - existsSync(join(root, "cmaps", "UniJIS-UTF16-H.bcmap")) && - existsSync(join(root, "wasm", "openjpeg.wasm")) && - assets.fallbackFonts.every((file) => existsSync(file)) - ) { - cachedAssets = assets; - } - } catch { - // Unresolvable: render without the data, as before, and say so below. - } - return cachedAssets; -} - let fallbackFamiliesRegistered = false; /** @@ -277,9 +154,8 @@ function inkRatio(context: PdfContext2D, width: number, height: number) { return pixels > 0 ? dark / pixels : 0; } -/** Test hook: the asset lookup and the font registration are per process. */ -export function __resetPdfjsAssetsForTests(): void { - cachedAssets = undefined; +/** Test hook: the font registration is per process. */ +export function __resetFallbackFamiliesForTests(): void { fallbackFamiliesRegistered = false; } @@ -386,11 +262,6 @@ export async function rasterizePdf( return { ok: false, reason: "unsupported" }; } try { - // Lazy import: keeps pdfjs (DOMMatrix) + @napi-rs/canvas out of the eager - // server chunk. Resolved at first rasterization, then module-cached. - const pdfjs = - (await import("pdfjs-dist/legacy/build/pdf.mjs")) as unknown as PdfjsModule; - const assets = resolvePdfjsAssets(); if (assets) { await registerFallbackFamilies(assets); @@ -401,22 +272,7 @@ export async function rasterizePdf( }); } - task = pdfjs.getDocument({ - data: new Uint8Array(buffer), - verbosity: 0, // VerbosityLevel.ERRORS — no console spam on odd PDFs. - isEvalSupported: false, // never eval font programs (defence in depth). - // There are no system fonts to use: substitutes come from pdfjs' own - // standard font data and the fallback families registered above. - useSystemFonts: false, - ...(assets - ? { - standardFontDataUrl: assets.standardFontDataUrl, - cMapUrl: assets.cMapUrl, - cMapPacked: true, - wasmUrl: assets.wasmUrl, - } - : {}), - }); + task = await openPdfDocument(buffer); doc = await task.promise; // Never exceed the standing DoS ceiling, even if a caller asks for more.