From bb762135dff5711d6b38a7bd858f897b15ef16df Mon Sep 17 00:00:00 2001 From: haritowa <6107211+haritowa@users.noreply.github.com> Date: Wed, 21 Oct 2026 01:35:59 +0000 Subject: [PATCH] fix(documents): read a PDF's text with the app's own pdfjs Local text extraction went through pdf-parse, which pins its own pdfjs-dist (5.4.296) beside the app's (6.4.299), and both copies take their worker from globalThis.pdfjsWorker. Whichever ran first in a process broke the other: after a text extraction every raster failed with "The API version "6.4.299" does not match the Worker version "5.4.296"" (thumbnails raster-failed, vision reads of a PDF 422 extractFailed), and after a raster the text extraction failed and the local content index stayed empty without a trace. extractPdfText now reads the text layer with getTextContent on the pdfjs-dist the rasterizer uses, so only one pdfjs runs in the server. Pointing pdf-parse at 6.4 instead is not enough: its destroy() calls PDFDocumentProxy.destroy(), which pdfjs 6 no longer has, so every document would skip its teardown. Two tests run text and raster in one process, each order in its own file. --- .../pdfjs-one-version-extract-first.test.ts | 40 ++++++ .../pdfjs-one-version-raster-first.test.ts | 40 ++++++ src/lib/documents/content-index.ts | 2 +- src/lib/documents/index-document.ts | 2 +- src/lib/documents/local-extract.ts | 119 ++++++++++++++---- 5 files changed, 176 insertions(+), 27 deletions(-) create mode 100644 src/lib/documents/__tests__/pdfjs-one-version-extract-first.test.ts create mode 100644 src/lib/documents/__tests__/pdfjs-one-version-raster-first.test.ts diff --git a/src/lib/documents/__tests__/pdfjs-one-version-extract-first.test.ts b/src/lib/documents/__tests__/pdfjs-one-version-extract-first.test.ts new file mode 100644 index 000000000..3b20f39da --- /dev/null +++ b/src/lib/documents/__tests__/pdfjs-one-version-extract-first.test.ts @@ -0,0 +1,40 @@ +import { readFileSync } from "node:fs"; +import { join } from "node:path"; + +import { describe, expect, it, vi } from "vitest"; + +/** + * Text extraction and rasterising share one process, and pdfjs keeps its worker + * on `globalThis.pdfjsWorker`. While `pdf-parse` brought its own pdfjs + * (5.4.296) beside the app's (6.4.299), whichever ran first installed its + * worker and the other refused it: "The API version "6.4.299" does not match + * the Worker version "5.4.296"". Thumbnails failed as `raster-failed`, vision + * reads of a PDF answered 422 and the local text index stayed empty without a + * word. Reading the text through the app's own pdfjs is the fix; this file runs + * extraction first and `pdfjs-one-version-raster-first.test.ts` the reverse, + * each in a fresh process state. + * + * Mutation check: read the text through `pdf-parse` again in `extractPdfText` → + * the raster step here fails. + */ +vi.mock("@/lib/logging/context", () => ({ annotate: vi.fn() })); + +import { extractPdfText } from "../local-extract"; +import { rasterizePdf } from "../rasterize-pdf"; + +const fixture = readFileSync( + join(process.cwd(), "tests", "fixtures", "pdf", "fallback-fonts.pdf"), +); + +describe("one pdfjs for text and raster: text first", () => { + it("extracts, rasterises and extracts again in one process", async () => { + const first = await extractPdfText(fixture); + expect(first.ok).toBe(true); + + const raster = await rasterizePdf(fixture, 1); + expect(raster.ok).toBe(true); + + const again = await extractPdfText(fixture); + expect(again).toEqual(first); + }); +}); diff --git a/src/lib/documents/__tests__/pdfjs-one-version-raster-first.test.ts b/src/lib/documents/__tests__/pdfjs-one-version-raster-first.test.ts new file mode 100644 index 000000000..3986da2fe --- /dev/null +++ b/src/lib/documents/__tests__/pdfjs-one-version-raster-first.test.ts @@ -0,0 +1,40 @@ +import { readFileSync } from "node:fs"; +import { join } from "node:path"; + +import { describe, expect, it, vi } from "vitest"; + +/** + * Text extraction and rasterising share one process, and pdfjs keeps its worker + * on `globalThis.pdfjsWorker`. While `pdf-parse` brought its own pdfjs + * (5.4.296) beside the app's (6.4.299), whichever ran first installed its + * worker and the other refused it: "The API version "6.4.299" does not match + * the Worker version "5.4.296"". Thumbnails failed as `raster-failed`, vision + * reads of a PDF answered 422 and the local text index stayed empty without a + * word. Reading the text through the app's own pdfjs is the fix; the twin runs + * extraction first (`pdfjs-one-version-extract-first.test.ts`) and this file + * the reverse, each in a fresh process state. + * + * Mutation check: read the text through `pdf-parse` again in `extractPdfText` → + * the text step here fails. + */ +vi.mock("@/lib/logging/context", () => ({ annotate: vi.fn() })); + +import { extractPdfText } from "../local-extract"; +import { rasterizePdf } from "../rasterize-pdf"; + +const fixture = readFileSync( + join(process.cwd(), "tests", "fixtures", "pdf", "fallback-fonts.pdf"), +); + +describe("one pdfjs for text and raster: raster first", () => { + it("rasterises, extracts and rasterises again in one process", async () => { + const first = await rasterizePdf(fixture, 1); + expect(first.ok).toBe(true); + + const text = await extractPdfText(fixture); + expect(text.ok).toBe(true); + + const again = await rasterizePdf(fixture, 1); + expect(again.ok).toBe(true); + }); +}); diff --git a/src/lib/documents/content-index.ts b/src/lib/documents/content-index.ts index 4b88616f3..d9f789186 100644 --- a/src/lib/documents/content-index.ts +++ b/src/lib/documents/content-index.ts @@ -325,7 +325,7 @@ export async function loadDocumentChatText( * - `vision` → an AI provider transcribed the stored original (image or PDF, * incl. scanned) — the AI-first primary path. * - `text-ocr` → browser-OCR text posted by the client (opt-in local OCR). - * - `local-pdf` → server-side text-layer extraction of a PDF (`pdf-parse`), no + * - `local-pdf` → server-side text-layer extraction of a PDF (pdfjs), no * provider, no egress — the fallback when no provider is usable. * - `local-ocr` → server-side OCR of a scanned image/PDF (deferred follow-up). */ diff --git a/src/lib/documents/index-document.ts b/src/lib/documents/index-document.ts index 80c3ff25d..aef4348de 100644 --- a/src/lib/documents/index-document.ts +++ b/src/lib/documents/index-document.ts @@ -24,7 +24,7 @@ * configured, the toggle is OFF for an external pick, the provider can't * read this file (e.g. a text-layer PDF on a non-Anthropic account), the * daily budget is reached, or the provider call fails — fall back to - * server-side text-layer extraction (`pdf-parse`, no egress, milliseconds). + * server-side text-layer extraction (pdfjs, no egress, milliseconds). * So content search works at a baseline even with no AI, and every * text-layer PDF is searchable regardless of provider. * diff --git a/src/lib/documents/local-extract.ts b/src/lib/documents/local-extract.ts index d473bd043..8476eb803 100644 --- a/src/lib/documents/local-extract.ts +++ b/src/lib/documents/local-extract.ts @@ -3,9 +3,17 @@ * * When NO AI provider is configured — or the user has not consented to one — * content search still needs a baseline. This module extracts a PDF's embedded - * text layer SERVER-SIDE with `pdf-parse` (already a direct dependency; its - * `getText()` touches only the pure-JS/wasm pdfjs core, never the native - * `@napi-rs/canvas`), in milliseconds, with no OCR and no third-party egress. + * text layer SERVER-SIDE with pdfjs' `getTextContent` (the pure-JS/wasm core, + * never the native `@napi-rs/canvas`), in milliseconds, with no OCR and no + * third-party egress. + * + * It reads through the app's own `pdfjs-dist`, the copy `./rasterize-pdf` + * renders with. It used to go through `pdf-parse`, which brings a second + * pdfjs of its own, and the two cannot share a process: both take their + * worker from `globalThis.pdfjsWorker`, so whichever ran first broke the + * other ("The API version "6.4.299" does not match the Worker version + * "5.4.296""). Thumbnails and vision reads failed after a text extraction, + * and the text extraction failed after a raster. * The extracted text feeds the SAME blind, encrypted content index the provider * path writes (`upsertContentIndex`), so nothing readable is stored at rest. * @@ -20,19 +28,68 @@ */ import { Buffer } from "node:buffer"; -// `pdf-parse` pulls in pdfjs, whose module top-level references the browser-only -// `DOMMatrix` global. A STATIC import makes the Turbopack server chunk evaluate -// that reference at instantiation and throw `DOMMatrix is not defined`, taking -// down every route/worker that shares the chunk (the document routes among -// them). The value is therefore loaded lazily inside `extractPdfText` via a -// runtime `import()`, which lets pdf-parse initialise pdfjs on its own terms and -// never touches DOMMatrix at load — verified to extract a real text layer under -// plain Node. Keep the surface type as a type-only import (fully erased at -// build, no runtime module evaluation). -import type { PDFParse } from "pdf-parse"; - import { annotate } from "@/lib/logging/context"; +// pdfjs' module top-level references the browser-only `DOMMatrix` global. A +// STATIC import makes the Turbopack server chunk evaluate that reference at +// instantiation and throw `DOMMatrix is not defined`, taking down every +// route/worker that shares the chunk (the document routes among them). The +// module is therefore loaded lazily inside `extractPdfText` via a runtime +// `import()`, and only the slice of its API used here is typed, structurally. +interface PdfTextItem { + str: string; + /** [a, b, c, d, x, y]: the run's text matrix; x and y place it on the page. */ + transform: number[]; + width: number; + height: number; + hasEOL: boolean; +} +interface PdfTextPage { + getTextContent(): Promise<{ items: Array }>; + cleanup(): void; +} +interface PdfTextDocument { + numPages: number; + getPage(pageNumber: number): Promise; +} +interface PdfTextLoadingTask { + promise: Promise; + destroy(): Promise; +} +interface PdfTextModule { + getDocument(opts: { + data: Uint8Array; + verbosity?: number; + isEvalSupported?: boolean; + useSystemFonts?: boolean; + }): PdfTextLoadingTask; +} + +/** + * One page's text runs as text: a new line where a run ends its line or the + * next one sits on another, a space where two runs on one line leave a gap, + * nothing where they touch (a word split across two fonts stays one word). + */ +function pageText(items: Array): string { + let text = ""; + let last: { end: number; y: number; size: number } | null = null; + for (const item of items) { + // Marked-content boundaries carry no text. + if (!("str" in item)) continue; + const x = item.transform[4] ?? 0; + const y = item.transform[5] ?? 0; + const size = Math.abs(item.transform[3] ?? 0) || item.height || 1; + if (last && !text.endsWith("\n")) { + if (Math.abs(y - last.y) > last.size / 2) text += "\n"; + else if (x - last.end > size * 0.1 && !/\s$/.test(text)) text += " "; + } + text += item.str; + if (item.hasEOL) text += "\n"; + last = { end: x + item.width, y, size }; + } + return text; +} + /** * Minimum joined text length for a PDF to count as a real text-layer document. * Below this a PDF is treated as scanned/empty — a few stray glyphs from a form @@ -68,19 +125,31 @@ export type LocalExtractResult = export async function extractPdfText( buffer: Buffer, ): Promise { - let parser: PDFParse | null = null; + let task: PdfTextLoadingTask | null = null; try { // Lazy load: keeps pdfjs/DOMMatrix out of the eager server chunk (see the // import-site note). Resolved at first PDF extraction, then module-cached. - const { PDFParse } = await import("pdf-parse"); - parser = new PDFParse({ data: new Uint8Array(buffer) }); - const result = await parser.getText({ - // Bound the page walk and drop the default "-- N of M --" page markers so - // only real document text is tokenised. - first: LOCAL_MAX_PAGES, - pageJoiner: "", + const pdfjs = + (await import("pdfjs-dist/legacy/build/pdf.mjs")) as unknown as PdfTextModule; + task = pdfjs.getDocument({ + data: new Uint8Array(buffer), + verbosity: 0, // VerbosityLevel.ERRORS — no console spam on odd PDFs. + isEvalSupported: false, // never eval font programs (defence in depth). + useSystemFonts: false, }); - const text = (result.text ?? "").trim(); + const doc = await task.promise; + // Bound the page walk so a pathological many-page PDF costs no more. + const pageCount = Math.min(doc.numPages, LOCAL_MAX_PAGES); + const pages: string[] = []; + for (let n = 1; n <= pageCount; n++) { + const page = await doc.getPage(n); + try { + pages.push(pageText((await page.getTextContent()).items)); + } finally { + page.cleanup(); + } + } + const text = pages.join("\n").trim(); if (text.length < LOCAL_TEXT_MIN_CHARS) { return { ok: false, reason: "empty" }; } @@ -88,9 +157,9 @@ export async function extractPdfText( } catch { return { ok: false, reason: "error" }; } finally { - if (parser) { + if (task) { try { - await parser.destroy(); + await task.destroy(); } catch { // Best-effort teardown; a destroy failure must not mask the result. }