diff --git a/src/__tests__/single-pdfjs-guard.test.ts b/src/__tests__/single-pdfjs-guard.test.ts index 38309e2b4..d42270995 100644 --- a/src/__tests__/single-pdfjs-guard.test.ts +++ b/src/__tests__/single-pdfjs-guard.test.ts @@ -1,172 +1,76 @@ /** - * Structural guard: the server runs one pdf.js, and only one module opens it. + * Structural guard: the server runs one pdfjs. * - * ## The failure this freezes (GitHub #1277) + * ## The failure this freezes * - * Under Node, pdf.js keeps its in-process worker on `globalThis.pdfjsWorker`, - * and whichever copy opens a document first owns it until the process exits. - * The local text read ran a library that pinned its own, older pdfjs-dist, - * while the rasterizer ran the app's. Whichever ran first broke the other - * with "The API version … does not match the Worker version …": after a text - * read every thumbnail and every vision read of a scan failed, after a render - * every local text read failed without a word, until a restart. Nothing in - * the gate noticed, because each path passed on its own and the image tracing - * shipped every `pdfjs-dist@*` the store held. + * `pdf-parse` brings its own `pdfjs-dist` beside the app's, and under Node + * every pdfjs copy keeps its worker on the one `globalThis.pdfjsWorker`. While + * local text extraction went through pdf-parse, whichever copy ran first in a + * process broke the other: "The API version "6.4.299" does not match the + * Worker version "5.4.296"". Thumbnails and vision reads failed after a text + * extraction, and the text extraction failed after a raster. * * ## What it proves and what it does not * - * - The lockfile resolves exactly one pdfjs-dist, the version package.json - * pins, and no package in it depends on pdfjs-dist except the app itself. - * A new dependency that brings its own pdf.js, even the same version, - * fails here and has to be looked at; every line of the lockfile that - * names pdfjs-dist is either understood or reported. - * - The checkout resolves that same version from the working directory, - * which is how the server finds it. - * - No production module under `src/` imports pdfjs-dist except - * `src/lib/documents/pdfjs.ts`, so both paths open documents with the same - * options, and nothing imports pdf-parse. - * - * It does not prove the image carries one copy; the Dockerfile hoists the - * pinned version by name and the tracing glob can only find what the lockfile - * installed. The behaviour itself is pinned by the two - * `pdfjs-one-runtime-*` tests under `src/lib/documents/__tests__/`. + * No runtime file under `src/` imports `pdf-parse`, so the only pdfjs the + * server loads is the app's own. Tests may still read generated PDFs back with + * it; they run in their own processes. The pdfjs-one-version tests prove the + * two orders in one process; this file stops the import from coming back. */ -import { createRequire } from "node:module"; import { join } from "node:path"; import { describe, expect, it } from "vitest"; -import { readSourceFolder, readSourceText } from "./helpers/source-files"; +import { + readSourceText, + requireFloor, + stripComments, + walkSourceFiles, +} from "./helpers/source-files"; -const ROOT = process.cwd(); -/** Relative to `src/`. */ -const LOADER = "lib/documents/pdfjs.ts"; +const SRC = join(process.cwd(), "src"); -const pkg = JSON.parse(readSourceText(join(ROOT, "package.json"))) as { - dependencies?: Record; - devDependencies?: Record; -}; -const pinned = - pkg.dependencies?.["pdfjs-dist"] ?? pkg.devDependencies?.["pdfjs-dist"]; - -/** Every lockfile line naming pdfjs-dist, sorted into what it means. */ -function readLockfile(lockfile: string): { - versions: Set; - rootImporters: number; - unexplained: string[]; -} { - const lines = lockfile.split("\n"); - const versions = new Set(); - let rootImporters = 0; - const unexplained: string[] = []; - lines.forEach((line, i) => { - if (!line.includes("pdfjs-dist")) return; - // A resolved copy: ` pdfjs-dist@6.4.299:` in `packages:` and again in - // `snapshots:` (where a peer suffix in parentheses may follow). - const resolved = - /^ {2}'?pdfjs-dist@(\d+\.\d+\.\d+[\w.+-]*)(?:\(.*\))?'?:/.exec(line); - if (resolved) { - versions.add(resolved[1]!); - return; - } - // The app's own entry under `importers:`, followed by its specifier. - if ( - /^ {6}pdfjs-dist:\s*$/.test(line) && - /^\s+specifier:/.test(lines[i + 1] ?? "") - ) { - rootImporters++; - return; - } - // Anything else is another package depending on pdfjs-dist, an - // override, or a spelling this reader does not know: all reported. - unexplained.push(`${i + 1}: ${line.trim()}`); - }); - return { versions, rootImporters, unexplained }; -} - -/** Production modules under `src/` that import a pdf.js package. */ -function pdfjsImporters(files: Map): string[] { - const specifier = - /(?:\bfrom\s*|\bimport\s*\(\s*|\brequire\s*\(\s*|^\s*import\s*)["'](?:pdfjs-dist|pdf-parse)(?:\/[^"']*)?["']/m; - return [...files] - .filter(([, source]) => specifier.test(source)) - .map(([path]) => path) - .sort(); -} - -/** Production modules under `src/`, keyed by their path relative to it. */ -function productionSources(): Map { - return new Map( - readSourceFolder("src", { - floor: 1000, - extensions: [".ts", ".tsx", ".mts", ".js", ".mjs"], - include: (rel) => - !rel.startsWith("generated/") && - !rel.includes("__tests__/") && - !/\.(?:test|spec)\.[mc]?[tj]sx?$/.test(rel), - }), +/** + * Every runtime module under `src/`: the suites, mocks and generated client + * are not loaded by the server. About 3,400 today; the floor sits below that + * with headroom, so a walk that comes back short fails instead of passing. + */ +function runtimeFiles(): string[] { + const files = walkSourceFiles(SRC, { + floor: 5000, + extensions: [".ts", ".tsx", ".js", ".jsx", ".mjs", ".cjs", ".mts", ".cts"], + }).filter( + (rel) => + !rel.startsWith("generated/") && + !rel.split("/").includes("__tests__") && + !rel.split("/").includes("__mocks__") && + !/\.test\.(ts|tsx)$/.test(rel), ); + return [...requireFloor("runtime files under src/", files, 3000)]; } -describe("one pdf.js (#1277)", () => { - it("package.json pins pdfjs-dist exactly and lists no pdf-parse", () => { - expect(pinned, "pdfjs-dist is a direct dependency").toMatch( - /^\d+\.\d+\.\d+$/, +/** A static import, a re-export, a dynamic import or a require of pdf-parse. */ +const PDF_PARSE_IMPORT = + /(?:\bfrom\s*|\bimport\s*\(\s*|\brequire\s*\(\s*|\bimport\s+)["']pdf-parse(?:\/[^"']*)?["']/; + +describe("one pdfjs in the server", () => { + it("no runtime file imports pdf-parse", () => { + const importers = runtimeFiles().filter((rel) => + PDF_PARSE_IMPORT.test(stripComments(readSourceText(join(SRC, rel)))), ); - expect(pkg.dependencies?.["pdf-parse"]).toBeUndefined(); - expect(pkg.devDependencies?.["pdf-parse"]).toBeUndefined(); + expect(importers).toEqual([]); }); - it("the lockfile resolves one pdfjs-dist, and only the app depends on it", () => { - const lock = readLockfile(readSourceText(join(ROOT, "pnpm-lock.yaml"))); - expect(lock.unexplained, "lockfile lines naming pdfjs-dist").toEqual([]); - expect([...lock.versions]).toEqual([pinned]); - expect(lock.rootImporters).toBe(1); - }); - - it("the checkout resolves the pinned copy", () => { - const anchor = createRequire(join(ROOT, "package.json")); - const resolved = JSON.parse( - readSourceText(anchor.resolve("pdfjs-dist/package.json")), - ) as { version: string }; - expect(resolved.version).toBe(pinned); - }); - - it("only the shared loader imports pdf.js", () => { - const sources = productionSources(); - expect(sources.has(LOADER), `${LOADER} exists`).toBe(true); - expect(pdfjsImporters(sources)).toEqual([LOADER]); - }); - - it("the readers catch what they exist to catch", () => { - const twoCopies = [ - "importers:", - " .:", - " devDependencies:", - " pdfjs-dist:", - " specifier: 6.4.299", - " version: 6.4.299", - "packages:", - " pdfjs-dist@5.4.296:", - " pdfjs-dist@6.4.299:", - "snapshots:", - " some-parser@2.4.5:", - " dependencies:", - " pdfjs-dist: 5.4.296", - ].join("\n"); - const lock = readLockfile(twoCopies); - expect([...lock.versions].sort()).toEqual(["5.4.296", "6.4.299"]); - expect(lock.unexplained).toEqual(["13: pdfjs-dist: 5.4.296"]); - - const importers = pdfjsImporters( - new Map([ - [LOADER, 'await import("pdfjs-dist/legacy/build/pdf.mjs")'], - ["a", 'import { PDFParse } from "pdf-parse";'], - ["b", "const m = await import(\n 'pdfjs-dist'\n);"], - ["c", 'import "pdfjs-dist/legacy/build/pdf.worker.mjs";'], - ["d", '// mentions "pdfjs-dist" in prose only'], - ]), - ); - expect(importers).toEqual(["a", "b", "c", LOADER].sort()); + it("recognises each way of importing it", () => { + for (const line of [ + 'import { PDFParse } from "pdf-parse";', + 'import type { PDFParse } from "pdf-parse";', + 'const { PDFParse } = await import("pdf-parse");', + "const m = require('pdf-parse/worker');", + 'export { PDFParse } from "pdf-parse";', + ]) { + expect(PDF_PARSE_IMPORT.test(line), line).toBe(true); + } + expect(PDF_PARSE_IMPORT.test('import x from "pdf-parser";')).toBe(false); }); }); diff --git a/src/lib/documents/__tests__/pdfjs-one-version-extract-first.test.ts b/src/lib/documents/__tests__/pdfjs-one-version-extract-first.test.ts new file mode 100644 index 000000000..298696510 --- /dev/null +++ b/src/lib/documents/__tests__/pdfjs-one-version-extract-first.test.ts @@ -0,0 +1,43 @@ +import { readFileSync } from "node:fs"; +import { join } from "node:path"; + +import { describe, expect, it, vi } from "vitest"; + +/** + * Text extraction and rasterising run in one process. In Node, pdfjs keeps its + * worker in one global, `globalThis.pdfjsWorker`. When `pdf-parse` brought its + * own pdfjs (5.4.296) beside the app's (6.4.299), the copy that ran first set + * that global, and the other copy refused it: "The API version "6.4.299" does + * not match the Worker version "5.4.296"". Thumbnails then failed as + * `raster-failed`, vision reads of a PDF answered 422, and the local text index + * stayed empty with no log line. The fix: read the text with the app's own + * pdfjs. + * + * This file extracts text first and rasterises second. Its twin in this + * folder runs the reverse order. Each order needs its own file, because a + * process keeps the first worker it loads. + * + * Mutation check: read the text through `pdf-parse` again in + * `extractPdfText` → the raster step here fails. + */ +vi.mock("@/lib/logging/context", () => ({ annotate: vi.fn() })); + +import { extractPdfText } from "../local-extract"; +import { rasterizePdf } from "../rasterize-pdf"; + +const fixture = readFileSync( + join(process.cwd(), "tests", "fixtures", "pdf", "fallback-fonts.pdf"), +); + +describe("one pdfjs for text and raster: text first", () => { + it("extracts, rasterises and extracts again in one process", async () => { + const first = await extractPdfText(fixture); + expect(first.ok).toBe(true); + + const raster = await rasterizePdf(fixture, 1); + expect(raster.ok).toBe(true); + + const again = await extractPdfText(fixture); + expect(again).toEqual(first); + }); +}); diff --git a/src/lib/documents/__tests__/pdfjs-one-version-raster-first.test.ts b/src/lib/documents/__tests__/pdfjs-one-version-raster-first.test.ts new file mode 100644 index 000000000..3389f078e --- /dev/null +++ b/src/lib/documents/__tests__/pdfjs-one-version-raster-first.test.ts @@ -0,0 +1,43 @@ +import { readFileSync } from "node:fs"; +import { join } from "node:path"; + +import { describe, expect, it, vi } from "vitest"; + +/** + * Text extraction and rasterising run in one process. In Node, pdfjs keeps its + * worker in one global, `globalThis.pdfjsWorker`. When `pdf-parse` brought its + * own pdfjs (5.4.296) beside the app's (6.4.299), the copy that ran first set + * that global, and the other copy refused it: "The API version "6.4.299" does + * not match the Worker version "5.4.296"". Thumbnails then failed as + * `raster-failed`, vision reads of a PDF answered 422, and the local text index + * stayed empty with no log line. The fix: read the text with the app's own + * pdfjs. + * + * This file rasterises first and extracts text second. Its twin in this + * folder runs the reverse order. Each order needs its own file, because a + * process keeps the first worker it loads. + * + * Mutation check: read the text through `pdf-parse` again in + * `extractPdfText` → the text step here fails. + */ +vi.mock("@/lib/logging/context", () => ({ annotate: vi.fn() })); + +import { extractPdfText } from "../local-extract"; +import { rasterizePdf } from "../rasterize-pdf"; + +const fixture = readFileSync( + join(process.cwd(), "tests", "fixtures", "pdf", "fallback-fonts.pdf"), +); + +describe("one pdfjs for text and raster: raster first", () => { + it("rasterises, extracts and rasterises again in one process", async () => { + const first = await rasterizePdf(fixture, 1); + expect(first.ok).toBe(true); + + const text = await extractPdfText(fixture); + expect(text.ok).toBe(true); + + const again = await rasterizePdf(fixture, 1); + expect(again.ok).toBe(true); + }); +}); diff --git a/src/lib/documents/content-index.ts b/src/lib/documents/content-index.ts index 8690260d7..d9f789186 100644 --- a/src/lib/documents/content-index.ts +++ b/src/lib/documents/content-index.ts @@ -325,7 +325,7 @@ export async function loadDocumentChatText( * - `vision` → an AI provider transcribed the stored original (image or PDF, * incl. scanned) — the AI-first primary path. * - `text-ocr` → browser-OCR text posted by the client (opt-in local OCR). - * - `local-pdf` → server-side text-layer extraction of a PDF (pdf.js), no + * - `local-pdf` → server-side text-layer extraction of a PDF (pdfjs), no * provider, no egress — the fallback when no provider is usable. * - `local-ocr` → server-side OCR of a scanned image/PDF (deferred follow-up). */ diff --git a/src/lib/documents/index-document.ts b/src/lib/documents/index-document.ts index 60afab046..aef4348de 100644 --- a/src/lib/documents/index-document.ts +++ b/src/lib/documents/index-document.ts @@ -24,7 +24,7 @@ * configured, the toggle is OFF for an external pick, the provider can't * read this file (e.g. a text-layer PDF on a non-Anthropic account), the * daily budget is reached, or the provider call fails — fall back to - * server-side text-layer extraction (pdf.js, no egress, milliseconds). + * server-side text-layer extraction (pdfjs, no egress, milliseconds). * So content search works at a baseline even with no AI, and every * text-layer PDF is searchable regardless of provider. * diff --git a/src/lib/documents/local-extract.ts b/src/lib/documents/local-extract.ts index b41598bc6..e726f4a27 100644 --- a/src/lib/documents/local-extract.ts +++ b/src/lib/documents/local-extract.ts @@ -3,9 +3,17 @@ * * When NO AI provider is configured — or the user has not consented to one — * content search still needs a baseline. This module extracts a PDF's embedded - * text layer SERVER-SIDE through the app's one pdf.js (`./pdfjs`; reading the - * text layer never draws, so the native canvas does no work here), in - * milliseconds, with no OCR and no third-party egress. + * text layer SERVER-SIDE with pdfjs' `getTextContent` (the pure-JS/wasm core, + * never the native `@napi-rs/canvas`), in milliseconds, with no OCR and no + * third-party egress. + * + * It reads through the app's own `pdfjs-dist`, the copy `./rasterize-pdf` + * renders with. It used to go through `pdf-parse`, which brings a second + * pdfjs of its own, and the two cannot share a process: both take their + * worker from `globalThis.pdfjsWorker`, so whichever ran first broke the + * other ("The API version "6.4.299" does not match the Worker version + * "5.4.296""). Thumbnails and vision reads failed after a text extraction, + * and the text extraction failed after a raster. * The extracted text feeds the SAME blind, encrypted content index the provider * path writes (`upsertContentIndex`), so nothing readable is stored at rest. * @@ -20,10 +28,92 @@ */ import { Buffer } from "node:buffer"; -import { annotate } from "@/lib/logging/context"; -import { logCaught } from "@/lib/logging/signal"; +import type { PDFDocumentLoadingTask } from "pdfjs-dist"; +import type { + TextItem, + TextMarkedContent, +} from "pdfjs-dist/types/src/display/api"; -import { readPdfText } from "./pdfjs"; +import { annotate } from "@/lib/logging/context"; + +// pdfjs' module top-level references the browser-only `DOMMatrix` global. A +// STATIC import makes the Turbopack server chunk evaluate that reference at +// instantiation and throw `DOMMatrix is not defined`, taking down every +// route/worker that shares the chunk (the document routes among them). The +// module is therefore loaded lazily inside `extractPdfText` via a runtime +// `import()`. The type imports above are erased at build, so they load +// nothing; they let the compiler check this file against pdfjs' own API. + +/** + * Where a text run sits on the page. pdfjs gives each run a text matrix + * `[a, b, c, d, x, y]`: `x` and `y` are where the run starts, and `d` is its + * vertical scale, which for upright text is the font size. + */ +interface RunPosition { + x: number; + /** The baseline the run sits on. */ + y: number; + /** Where the run ends: its start plus its width. */ + end: number; + fontSize: number; +} + +function positionOf(run: TextItem): RunPosition { + const [, , , scaleY, x = 0, y = 0] = run.transform as number[]; + return { + x, + y, + end: x + run.width, + fontSize: Math.abs(scaleY ?? 0) || run.height || 1, + }; +} + +/** + * The next run is on a new line when its baseline moved by more than half a + * font size. Runs on one line can sit a little higher or lower (a superscript, + * a mixed font), so a small shift does not count. + */ +function startsNewLine(previous: RunPosition, next: RunPosition): boolean { + return Math.abs(next.y - previous.y) > previous.fontSize / 2; +} + +/** + * Two runs on one line are separate words when the gap between them is wider + * than a tenth of the font size. Runs that touch, such as a word drawn in two + * fonts, stay one word. + */ +function leavesWordGap(previous: RunPosition, next: RunPosition): boolean { + return next.x - previous.end > next.fontSize / 10; +} + +/** + * One page's text: its runs in order, with a line break where a run ends a + * line or the next one starts a new line, and a space where two runs on one + * line leave a gap between words. + */ +function pageText(items: Array): string { + let text = ""; + let previous: RunPosition | null = null; + + for (const item of items) { + // Marked-content markers carry no text. + if (!("str" in item)) continue; + const position = positionOf(item); + + if (previous && !text.endsWith("\n")) { + if (startsNewLine(previous, position)) { + text += "\n"; + } else if (leavesWordGap(previous, position) && !/\s$/.test(text)) { + text += " "; + } + } + text += item.str; + if (item.hasEOL && !text.endsWith("\n")) text += "\n"; + + previous = position; + } + return text; +} /** * Minimum joined text length for a PDF to count as a real text-layer document. @@ -55,34 +145,49 @@ export type LocalExtractResult = * Extract a text-layer PDF's embedded text server-side. Returns `empty` for a * scanned/no-text-layer PDF and `error` for a malformed/encrypted one — never * throws, so a bad document can never abort the caller (the upload/backfill job - * just leaves it un-indexed). An `error` leaves a `warn` line naming what - * pdf.js threw, because the outcome alone does not say why a document stayed - * out of the index. + * just leaves it un-indexed). */ export async function extractPdfText( buffer: Buffer, ): Promise { - let raw: string; + let task: PDFDocumentLoadingTask | null = null; try { - // Bounded page walk; pages joined by a blank line, with no page markers, - // so only real document text is tokenised. - raw = await readPdfText(buffer, LOCAL_MAX_PAGES); - } catch (err) { - const errorType = err instanceof Error ? err.name || "Error" : typeof err; - annotate({ - action: { name: "documents.localIndex.extractFailed" }, - meta: { localExtractError: errorType }, - }); - logCaught("documents.local_extract.failed", err, { - bytes: buffer.length, + // Lazy load: keeps pdfjs/DOMMatrix out of the eager server chunk (see the + // import-site note). Resolved at first PDF extraction, then module-cached. + const pdfjs = await import("pdfjs-dist/legacy/build/pdf.mjs"); + task = pdfjs.getDocument({ + data: new Uint8Array(buffer), + verbosity: 0, // VerbosityLevel.ERRORS — no console spam on odd PDFs. + useSystemFonts: false, }); + const doc = await task.promise; + // Bound the page walk so a pathological many-page PDF costs no more. + const pageCount = Math.min(doc.numPages, LOCAL_MAX_PAGES); + const pages: string[] = []; + for (let n = 1; n <= pageCount; n++) { + const page = await doc.getPage(n); + try { + pages.push(pageText((await page.getTextContent()).items)); + } finally { + page.cleanup(); + } + } + const text = pages.join("\n").trim(); + if (text.length < LOCAL_TEXT_MIN_CHARS) { + return { ok: false, reason: "empty" }; + } + return { ok: true, text, source: "local-pdf" }; + } catch { return { ok: false, reason: "error" }; + } finally { + if (task) { + try { + await task.destroy(); + } catch { + // Best-effort teardown; a destroy failure must not mask the result. + } + } } - const text = raw.trim(); - if (text.length < LOCAL_TEXT_MIN_CHARS) { - return { ok: false, reason: "empty" }; - } - return { ok: true, text, source: "local-pdf" }; } /**