From bb762135dff5711d6b38a7bd858f897b15ef16df Mon Sep 17 00:00:00 2001 From: haritowa <6107211+haritowa@users.noreply.github.com> Date: Wed, 21 Oct 2026 01:35:59 +0000 Subject: [PATCH 1/9] fix(documents): read a PDF's text with the app's own pdfjs Local text extraction went through pdf-parse, which pins its own pdfjs-dist (5.4.296) beside the app's (6.4.299), and both copies take their worker from globalThis.pdfjsWorker. Whichever ran first in a process broke the other: after a text extraction every raster failed with "The API version "6.4.299" does not match the Worker version "5.4.296"" (thumbnails raster-failed, vision reads of a PDF 422 extractFailed), and after a raster the text extraction failed and the local content index stayed empty without a trace. extractPdfText now reads the text layer with getTextContent on the pdfjs-dist the rasterizer uses, so only one pdfjs runs in the server. Pointing pdf-parse at 6.4 instead is not enough: its destroy() calls PDFDocumentProxy.destroy(), which pdfjs 6 no longer has, so every document would skip its teardown. Two tests run text and raster in one process, each order in its own file. --- .../pdfjs-one-version-extract-first.test.ts | 40 ++++++ .../pdfjs-one-version-raster-first.test.ts | 40 ++++++ src/lib/documents/content-index.ts | 2 +- src/lib/documents/index-document.ts | 2 +- src/lib/documents/local-extract.ts | 119 ++++++++++++++---- 5 files changed, 176 insertions(+), 27 deletions(-) create mode 100644 src/lib/documents/__tests__/pdfjs-one-version-extract-first.test.ts create mode 100644 src/lib/documents/__tests__/pdfjs-one-version-raster-first.test.ts diff --git a/src/lib/documents/__tests__/pdfjs-one-version-extract-first.test.ts b/src/lib/documents/__tests__/pdfjs-one-version-extract-first.test.ts new file mode 100644 index 000000000..3b20f39da --- /dev/null +++ b/src/lib/documents/__tests__/pdfjs-one-version-extract-first.test.ts @@ -0,0 +1,40 @@ +import { readFileSync } from "node:fs"; +import { join } from "node:path"; + +import { describe, expect, it, vi } from "vitest"; + +/** + * Text extraction and rasterising share one process, and pdfjs keeps its worker + * on `globalThis.pdfjsWorker`. While `pdf-parse` brought its own pdfjs + * (5.4.296) beside the app's (6.4.299), whichever ran first installed its + * worker and the other refused it: "The API version "6.4.299" does not match + * the Worker version "5.4.296"". Thumbnails failed as `raster-failed`, vision + * reads of a PDF answered 422 and the local text index stayed empty without a + * word. Reading the text through the app's own pdfjs is the fix; this file runs + * extraction first and `pdfjs-one-version-raster-first.test.ts` the reverse, + * each in a fresh process state. + * + * Mutation check: read the text through `pdf-parse` again in `extractPdfText` → + * the raster step here fails. + */ +vi.mock("@/lib/logging/context", () => ({ annotate: vi.fn() })); + +import { extractPdfText } from "../local-extract"; +import { rasterizePdf } from "../rasterize-pdf"; + +const fixture = readFileSync( + join(process.cwd(), "tests", "fixtures", "pdf", "fallback-fonts.pdf"), +); + +describe("one pdfjs for text and raster: text first", () => { + it("extracts, rasterises and extracts again in one process", async () => { + const first = await extractPdfText(fixture); + expect(first.ok).toBe(true); + + const raster = await rasterizePdf(fixture, 1); + expect(raster.ok).toBe(true); + + const again = await extractPdfText(fixture); + expect(again).toEqual(first); + }); +}); diff --git a/src/lib/documents/__tests__/pdfjs-one-version-raster-first.test.ts b/src/lib/documents/__tests__/pdfjs-one-version-raster-first.test.ts new file mode 100644 index 000000000..3986da2fe --- /dev/null +++ b/src/lib/documents/__tests__/pdfjs-one-version-raster-first.test.ts @@ -0,0 +1,40 @@ +import { readFileSync } from "node:fs"; +import { join } from "node:path"; + +import { describe, expect, it, vi } from "vitest"; + +/** + * Text extraction and rasterising share one process, and pdfjs keeps its worker + * on `globalThis.pdfjsWorker`. While `pdf-parse` brought its own pdfjs + * (5.4.296) beside the app's (6.4.299), whichever ran first installed its + * worker and the other refused it: "The API version "6.4.299" does not match + * the Worker version "5.4.296"". Thumbnails failed as `raster-failed`, vision + * reads of a PDF answered 422 and the local text index stayed empty without a + * word. Reading the text through the app's own pdfjs is the fix; the twin runs + * extraction first (`pdfjs-one-version-extract-first.test.ts`) and this file + * the reverse, each in a fresh process state. + * + * Mutation check: read the text through `pdf-parse` again in `extractPdfText` → + * the text step here fails. + */ +vi.mock("@/lib/logging/context", () => ({ annotate: vi.fn() })); + +import { extractPdfText } from "../local-extract"; +import { rasterizePdf } from "../rasterize-pdf"; + +const fixture = readFileSync( + join(process.cwd(), "tests", "fixtures", "pdf", "fallback-fonts.pdf"), +); + +describe("one pdfjs for text and raster: raster first", () => { + it("rasterises, extracts and rasterises again in one process", async () => { + const first = await rasterizePdf(fixture, 1); + expect(first.ok).toBe(true); + + const text = await extractPdfText(fixture); + expect(text.ok).toBe(true); + + const again = await rasterizePdf(fixture, 1); + expect(again.ok).toBe(true); + }); +}); diff --git a/src/lib/documents/content-index.ts b/src/lib/documents/content-index.ts index 4b88616f3..d9f789186 100644 --- a/src/lib/documents/content-index.ts +++ b/src/lib/documents/content-index.ts @@ -325,7 +325,7 @@ export async function loadDocumentChatText( * - `vision` → an AI provider transcribed the stored original (image or PDF, * incl. scanned) — the AI-first primary path. * - `text-ocr` → browser-OCR text posted by the client (opt-in local OCR). - * - `local-pdf` → server-side text-layer extraction of a PDF (`pdf-parse`), no + * - `local-pdf` → server-side text-layer extraction of a PDF (pdfjs), no * provider, no egress — the fallback when no provider is usable. * - `local-ocr` → server-side OCR of a scanned image/PDF (deferred follow-up). */ diff --git a/src/lib/documents/index-document.ts b/src/lib/documents/index-document.ts index 80c3ff25d..aef4348de 100644 --- a/src/lib/documents/index-document.ts +++ b/src/lib/documents/index-document.ts @@ -24,7 +24,7 @@ * configured, the toggle is OFF for an external pick, the provider can't * read this file (e.g. a text-layer PDF on a non-Anthropic account), the * daily budget is reached, or the provider call fails — fall back to - * server-side text-layer extraction (`pdf-parse`, no egress, milliseconds). + * server-side text-layer extraction (pdfjs, no egress, milliseconds). * So content search works at a baseline even with no AI, and every * text-layer PDF is searchable regardless of provider. * diff --git a/src/lib/documents/local-extract.ts b/src/lib/documents/local-extract.ts index d473bd043..8476eb803 100644 --- a/src/lib/documents/local-extract.ts +++ b/src/lib/documents/local-extract.ts @@ -3,9 +3,17 @@ * * When NO AI provider is configured — or the user has not consented to one — * content search still needs a baseline. This module extracts a PDF's embedded - * text layer SERVER-SIDE with `pdf-parse` (already a direct dependency; its - * `getText()` touches only the pure-JS/wasm pdfjs core, never the native - * `@napi-rs/canvas`), in milliseconds, with no OCR and no third-party egress. + * text layer SERVER-SIDE with pdfjs' `getTextContent` (the pure-JS/wasm core, + * never the native `@napi-rs/canvas`), in milliseconds, with no OCR and no + * third-party egress. + * + * It reads through the app's own `pdfjs-dist`, the copy `./rasterize-pdf` + * renders with. It used to go through `pdf-parse`, which brings a second + * pdfjs of its own, and the two cannot share a process: both take their + * worker from `globalThis.pdfjsWorker`, so whichever ran first broke the + * other ("The API version "6.4.299" does not match the Worker version + * "5.4.296""). Thumbnails and vision reads failed after a text extraction, + * and the text extraction failed after a raster. * The extracted text feeds the SAME blind, encrypted content index the provider * path writes (`upsertContentIndex`), so nothing readable is stored at rest. * @@ -20,19 +28,68 @@ */ import { Buffer } from "node:buffer"; -// `pdf-parse` pulls in pdfjs, whose module top-level references the browser-only -// `DOMMatrix` global. A STATIC import makes the Turbopack server chunk evaluate -// that reference at instantiation and throw `DOMMatrix is not defined`, taking -// down every route/worker that shares the chunk (the document routes among -// them). The value is therefore loaded lazily inside `extractPdfText` via a -// runtime `import()`, which lets pdf-parse initialise pdfjs on its own terms and -// never touches DOMMatrix at load — verified to extract a real text layer under -// plain Node. Keep the surface type as a type-only import (fully erased at -// build, no runtime module evaluation). -import type { PDFParse } from "pdf-parse"; - import { annotate } from "@/lib/logging/context"; +// pdfjs' module top-level references the browser-only `DOMMatrix` global. A +// STATIC import makes the Turbopack server chunk evaluate that reference at +// instantiation and throw `DOMMatrix is not defined`, taking down every +// route/worker that shares the chunk (the document routes among them). The +// module is therefore loaded lazily inside `extractPdfText` via a runtime +// `import()`, and only the slice of its API used here is typed, structurally. +interface PdfTextItem { + str: string; + /** [a, b, c, d, x, y]: the run's text matrix; x and y place it on the page. */ + transform: number[]; + width: number; + height: number; + hasEOL: boolean; +} +interface PdfTextPage { + getTextContent(): Promise<{ items: Array }>; + cleanup(): void; +} +interface PdfTextDocument { + numPages: number; + getPage(pageNumber: number): Promise; +} +interface PdfTextLoadingTask { + promise: Promise; + destroy(): Promise; +} +interface PdfTextModule { + getDocument(opts: { + data: Uint8Array; + verbosity?: number; + isEvalSupported?: boolean; + useSystemFonts?: boolean; + }): PdfTextLoadingTask; +} + +/** + * One page's text runs as text: a new line where a run ends its line or the + * next one sits on another, a space where two runs on one line leave a gap, + * nothing where they touch (a word split across two fonts stays one word). + */ +function pageText(items: Array): string { + let text = ""; + let last: { end: number; y: number; size: number } | null = null; + for (const item of items) { + // Marked-content boundaries carry no text. + if (!("str" in item)) continue; + const x = item.transform[4] ?? 0; + const y = item.transform[5] ?? 0; + const size = Math.abs(item.transform[3] ?? 0) || item.height || 1; + if (last && !text.endsWith("\n")) { + if (Math.abs(y - last.y) > last.size / 2) text += "\n"; + else if (x - last.end > size * 0.1 && !/\s$/.test(text)) text += " "; + } + text += item.str; + if (item.hasEOL) text += "\n"; + last = { end: x + item.width, y, size }; + } + return text; +} + /** * Minimum joined text length for a PDF to count as a real text-layer document. * Below this a PDF is treated as scanned/empty — a few stray glyphs from a form @@ -68,19 +125,31 @@ export type LocalExtractResult = export async function extractPdfText( buffer: Buffer, ): Promise { - let parser: PDFParse | null = null; + let task: PdfTextLoadingTask | null = null; try { // Lazy load: keeps pdfjs/DOMMatrix out of the eager server chunk (see the // import-site note). Resolved at first PDF extraction, then module-cached. - const { PDFParse } = await import("pdf-parse"); - parser = new PDFParse({ data: new Uint8Array(buffer) }); - const result = await parser.getText({ - // Bound the page walk and drop the default "-- N of M --" page markers so - // only real document text is tokenised. - first: LOCAL_MAX_PAGES, - pageJoiner: "", + const pdfjs = + (await import("pdfjs-dist/legacy/build/pdf.mjs")) as unknown as PdfTextModule; + task = pdfjs.getDocument({ + data: new Uint8Array(buffer), + verbosity: 0, // VerbosityLevel.ERRORS — no console spam on odd PDFs. + isEvalSupported: false, // never eval font programs (defence in depth). + useSystemFonts: false, }); - const text = (result.text ?? "").trim(); + const doc = await task.promise; + // Bound the page walk so a pathological many-page PDF costs no more. + const pageCount = Math.min(doc.numPages, LOCAL_MAX_PAGES); + const pages: string[] = []; + for (let n = 1; n <= pageCount; n++) { + const page = await doc.getPage(n); + try { + pages.push(pageText((await page.getTextContent()).items)); + } finally { + page.cleanup(); + } + } + const text = pages.join("\n").trim(); if (text.length < LOCAL_TEXT_MIN_CHARS) { return { ok: false, reason: "empty" }; } @@ -88,9 +157,9 @@ export async function extractPdfText( } catch { return { ok: false, reason: "error" }; } finally { - if (parser) { + if (task) { try { - await parser.destroy(); + await task.destroy(); } catch { // Best-effort teardown; a destroy failure must not mask the result. } From 275121391b3341c0994268e38d840b8355ad4b5a Mon Sep 17 00:00:00 2001 From: haritowa <6107211+haritowa@users.noreply.github.com> Date: Thu, 22 Oct 2026 16:48:16 +0000 Subject: [PATCH 2/9] test(documents): keep pdf-parse out of the server's runtime A structural guard fails when any runtime file under src/ imports pdf-parse again, which would bring its second pdfjs back into the server process. The next.config comment on the tracing includes now describes the text path as it is, and the extractor no longer turns an empty end-of-line run into a blank line of its own. --- next.config.ts | 13 ++-- src/__tests__/single-pdfjs-guard.test.ts | 78 ++++++++++++++++++++++++ src/lib/documents/local-extract.ts | 5 +- 3 files changed, 90 insertions(+), 6 deletions(-) create mode 100644 src/__tests__/single-pdfjs-guard.test.ts diff --git a/next.config.ts b/next.config.ts index 055b1dcf7..66662bb82 100644 --- a/next.config.ts +++ b/next.config.ts @@ -302,11 +302,14 @@ const nextConfig: NextConfig = { outputFileTracingIncludes: { "*": [ "./src/lib/ai/prompts/safety-contracts.*.yaml", - // Document AI local text extraction runs `pdf-parse` (→ pdfjs-dist) - // server-side in the content-index job — the provider-free fallback that - // keeps content search working with no AI configured. Both are pure JS + - // wasm; there is NO native `.node` on the text-layer path (`@napi-rs/canvas` - // is only pulled for image rendering, which `getText()` never invokes). + // Document AI local text extraction reads the text layer with pdfjs-dist + // (`getTextContent`, `src/lib/documents/local-extract.ts`) server-side in + // the content-index job — the provider-free fallback that keeps content + // search working with no AI configured. It is pure JS + wasm; there is NO + // native `.node` on the text-layer path (`@napi-rs/canvas` is only pulled + // for image rendering). Nothing at runtime imports `pdf-parse` any more + // (its own older pdfjs clashed with the app's); its glob below is dead + // weight kept until the image is rebuilt and checked without it. // The standalone tracer can miss pdfjs-dist's wasm decoders (they resolve // via `new URL(..., import.meta.url)`), so pin both packages into the // runtime image explicitly. pnpm keeps pdfjs-dist unhoisted under `.pnpm`. diff --git a/src/__tests__/single-pdfjs-guard.test.ts b/src/__tests__/single-pdfjs-guard.test.ts new file mode 100644 index 000000000..897189e1b --- /dev/null +++ b/src/__tests__/single-pdfjs-guard.test.ts @@ -0,0 +1,78 @@ +/** + * Structural guard: the server runs one pdfjs. + * + * ## The failure this freezes + * + * `pdf-parse` brings its own `pdfjs-dist` beside the app's, and under Node + * every pdfjs copy keeps its worker on the one `globalThis.pdfjsWorker`. While + * local text extraction went through pdf-parse, whichever copy ran first in a + * process broke the other: "The API version "6.4.299" does not match the + * Worker version "5.4.296"". Thumbnails and vision reads failed after a text + * extraction, and the text extraction failed after a raster. + * + * ## What it proves and what it does not + * + * No runtime file under `src/` imports `pdf-parse`, so the only pdfjs the + * server loads is the app's own. Tests may still read generated PDFs back with + * it; they run in their own processes. The pdfjs-one-version tests prove the + * two orders in one process; this file stops the import from coming back. + */ +import { readFileSync } from "node:fs"; +import { join } from "node:path"; + +import { describe, expect, it } from "vitest"; + +import { + requireFloor, + stripComments, + walkSourceFiles, +} from "./helpers/source-files"; + +const SRC = join(process.cwd(), "src"); + +/** + * Every runtime module under `src/`: the suites, mocks and generated client + * are not loaded by the server. About 3,400 today; the floor sits below that + * with headroom, so a walk that comes back short fails instead of passing. + */ +function runtimeFiles(): string[] { + const files = walkSourceFiles(SRC, { + floor: 5000, + extensions: [".ts", ".tsx", ".js", ".jsx", ".mjs", ".cjs", ".mts", ".cts"], + }).filter( + (rel) => + !rel.startsWith("generated/") && + !rel.split("/").includes("__tests__") && + !rel.split("/").includes("__mocks__") && + !/\.test\.(ts|tsx)$/.test(rel), + ); + return [...requireFloor("runtime files under src/", files, 3000)]; +} + +/** A static import, a re-export, a dynamic import or a require of pdf-parse. */ +const PDF_PARSE_IMPORT = + /(?:\bfrom\s*|\bimport\s*\(\s*|\brequire\s*\(\s*|\bimport\s+)["']pdf-parse(?:\/[^"']*)?["']/; + +describe("one pdfjs in the server", () => { + it("no runtime file imports pdf-parse", () => { + const importers = runtimeFiles().filter((rel) => + PDF_PARSE_IMPORT.test( + stripComments(readFileSync(join(SRC, rel), "utf8")), + ), + ); + expect(importers).toEqual([]); + }); + + it("recognises each way of importing it", () => { + for (const line of [ + 'import { PDFParse } from "pdf-parse";', + 'import type { PDFParse } from "pdf-parse";', + 'const { PDFParse } = await import("pdf-parse");', + "const m = require('pdf-parse/worker');", + 'export { PDFParse } from "pdf-parse";', + ]) { + expect(PDF_PARSE_IMPORT.test(line), line).toBe(true); + } + expect(PDF_PARSE_IMPORT.test('import x from "pdf-parser";')).toBe(false); + }); +}); diff --git a/src/lib/documents/local-extract.ts b/src/lib/documents/local-extract.ts index 8476eb803..39b33df09 100644 --- a/src/lib/documents/local-extract.ts +++ b/src/lib/documents/local-extract.ts @@ -84,7 +84,10 @@ function pageText(items: Array): string { else if (x - last.end > size * 0.1 && !/\s$/.test(text)) text += " "; } text += item.str; - if (item.hasEOL) text += "\n"; + // An empty run that only ends a line adds no blank line of its own. + if (item.hasEOL && !(item.str === "" && text.endsWith("\n"))) { + text += "\n"; + } last = { end: x + item.width, y, size }; } return text; From 5090a6fdc1ad41d9aaac5edeb6bcad4da4c52460 Mon Sep 17 00:00:00 2001 From: haritowa <6107211+haritowa@users.noreply.github.com> Date: Fri, 23 Oct 2026 17:31:25 +0000 Subject: [PATCH 3/9] chore(deps): say what the pdf-parse copy of pdfjs is still for The override comment said pdf-parse's pdfjs 5.4.296 drives local text extraction. Extraction now reads with the app's own pdfjs; only the tests still use pdf-parse, which has to stay on pdfjs 5 because its teardown calls an API pdfjs 6 removed. The range itself is unchanged. --- pnpm-workspace.yaml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index 2f95c35f1..aade691c4 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -73,8 +73,10 @@ overrides: # transitively, and `outputFileTracingIncludes` ships EVERY `pdfjs-dist@*` in # the store into the standalone image — so an unused transitive copy is still # a copy in the runtime image. The range is bounded on both ends deliberately: - # `pdf-parse` resolves 5.4.296, which predates the flaw and drives the local - # text-extraction path, so it must not be dragged onto a new major. + # `pdf-parse` resolves 5.4.296, which predates the flaw. Only the tests use it + # now (they read generated PDFs back); it must not be dragged onto a new + # major, because its teardown calls `PDFDocumentProxy.destroy()`, which pdfjs + # 6 no longer has. Local text extraction reads with the app's own pdfjs. "pdfjs-dist@>=5.6.83 <6.2.108": "6.2.108" # Deeply nested input drives DeepmergeTS into unbounded recursion until the # stack gives out. The copy here is Prisma's: `@prisma/config` still asks for From 85bef667dd0d9d5052da948877090fb752f1088d Mon Sep 17 00:00:00 2001 From: haritowa <6107211+haritowa@users.noreply.github.com> Date: Fri, 23 Oct 2026 17:56:38 +0000 Subject: [PATCH 4/9] test(documents): read files the way the realism ratchet expects The test-realism ratchet counts a test that calls readFileSync and names a source file. The guard now reads source through readSourceText, and the two one-process tests no longer name their twin file in a comment. Their doc comments are rewritten in plainer words. --- src/__tests__/single-pdfjs-guard.test.ts | 6 ++--- .../pdfjs-one-version-extract-first.test.ts | 25 +++++++++++-------- .../pdfjs-one-version-raster-first.test.ts | 25 +++++++++++-------- 3 files changed, 30 insertions(+), 26 deletions(-) diff --git a/src/__tests__/single-pdfjs-guard.test.ts b/src/__tests__/single-pdfjs-guard.test.ts index 897189e1b..d42270995 100644 --- a/src/__tests__/single-pdfjs-guard.test.ts +++ b/src/__tests__/single-pdfjs-guard.test.ts @@ -17,12 +17,12 @@ * it; they run in their own processes. The pdfjs-one-version tests prove the * two orders in one process; this file stops the import from coming back. */ -import { readFileSync } from "node:fs"; import { join } from "node:path"; import { describe, expect, it } from "vitest"; import { + readSourceText, requireFloor, stripComments, walkSourceFiles, @@ -56,9 +56,7 @@ const PDF_PARSE_IMPORT = describe("one pdfjs in the server", () => { it("no runtime file imports pdf-parse", () => { const importers = runtimeFiles().filter((rel) => - PDF_PARSE_IMPORT.test( - stripComments(readFileSync(join(SRC, rel), "utf8")), - ), + PDF_PARSE_IMPORT.test(stripComments(readSourceText(join(SRC, rel)))), ); expect(importers).toEqual([]); }); diff --git a/src/lib/documents/__tests__/pdfjs-one-version-extract-first.test.ts b/src/lib/documents/__tests__/pdfjs-one-version-extract-first.test.ts index 3b20f39da..298696510 100644 --- a/src/lib/documents/__tests__/pdfjs-one-version-extract-first.test.ts +++ b/src/lib/documents/__tests__/pdfjs-one-version-extract-first.test.ts @@ -4,18 +4,21 @@ import { join } from "node:path"; import { describe, expect, it, vi } from "vitest"; /** - * Text extraction and rasterising share one process, and pdfjs keeps its worker - * on `globalThis.pdfjsWorker`. While `pdf-parse` brought its own pdfjs - * (5.4.296) beside the app's (6.4.299), whichever ran first installed its - * worker and the other refused it: "The API version "6.4.299" does not match - * the Worker version "5.4.296"". Thumbnails failed as `raster-failed`, vision - * reads of a PDF answered 422 and the local text index stayed empty without a - * word. Reading the text through the app's own pdfjs is the fix; this file runs - * extraction first and `pdfjs-one-version-raster-first.test.ts` the reverse, - * each in a fresh process state. + * Text extraction and rasterising run in one process. In Node, pdfjs keeps its + * worker in one global, `globalThis.pdfjsWorker`. When `pdf-parse` brought its + * own pdfjs (5.4.296) beside the app's (6.4.299), the copy that ran first set + * that global, and the other copy refused it: "The API version "6.4.299" does + * not match the Worker version "5.4.296"". Thumbnails then failed as + * `raster-failed`, vision reads of a PDF answered 422, and the local text index + * stayed empty with no log line. The fix: read the text with the app's own + * pdfjs. * - * Mutation check: read the text through `pdf-parse` again in `extractPdfText` → - * the raster step here fails. + * This file extracts text first and rasterises second. Its twin in this + * folder runs the reverse order. Each order needs its own file, because a + * process keeps the first worker it loads. + * + * Mutation check: read the text through `pdf-parse` again in + * `extractPdfText` → the raster step here fails. */ vi.mock("@/lib/logging/context", () => ({ annotate: vi.fn() })); diff --git a/src/lib/documents/__tests__/pdfjs-one-version-raster-first.test.ts b/src/lib/documents/__tests__/pdfjs-one-version-raster-first.test.ts index 3986da2fe..3389f078e 100644 --- a/src/lib/documents/__tests__/pdfjs-one-version-raster-first.test.ts +++ b/src/lib/documents/__tests__/pdfjs-one-version-raster-first.test.ts @@ -4,18 +4,21 @@ import { join } from "node:path"; import { describe, expect, it, vi } from "vitest"; /** - * Text extraction and rasterising share one process, and pdfjs keeps its worker - * on `globalThis.pdfjsWorker`. While `pdf-parse` brought its own pdfjs - * (5.4.296) beside the app's (6.4.299), whichever ran first installed its - * worker and the other refused it: "The API version "6.4.299" does not match - * the Worker version "5.4.296"". Thumbnails failed as `raster-failed`, vision - * reads of a PDF answered 422 and the local text index stayed empty without a - * word. Reading the text through the app's own pdfjs is the fix; the twin runs - * extraction first (`pdfjs-one-version-extract-first.test.ts`) and this file - * the reverse, each in a fresh process state. + * Text extraction and rasterising run in one process. In Node, pdfjs keeps its + * worker in one global, `globalThis.pdfjsWorker`. When `pdf-parse` brought its + * own pdfjs (5.4.296) beside the app's (6.4.299), the copy that ran first set + * that global, and the other copy refused it: "The API version "6.4.299" does + * not match the Worker version "5.4.296"". Thumbnails then failed as + * `raster-failed`, vision reads of a PDF answered 422, and the local text index + * stayed empty with no log line. The fix: read the text with the app's own + * pdfjs. * - * Mutation check: read the text through `pdf-parse` again in `extractPdfText` → - * the text step here fails. + * This file rasterises first and extracts text second. Its twin in this + * folder runs the reverse order. Each order needs its own file, because a + * process keeps the first worker it loads. + * + * Mutation check: read the text through `pdf-parse` again in + * `extractPdfText` → the text step here fails. */ vi.mock("@/lib/logging/context", () => ({ annotate: vi.fn() })); From 2c5e9a80b9ee6ba43dde7cd4166addc573c0d535 Mon Sep 17 00:00:00 2001 From: haritowa <6107211+haritowa@users.noreply.github.com> Date: Fri, 23 Oct 2026 18:35:17 +0000 Subject: [PATCH 5/9] refactor(documents): type the text read with pdfjs' own declarations extractPdfText cast the dynamic import to a hand-written shape (`as unknown as`), so the compiler could not check the call against pdfjs. It now uses pdfjs' own types: PDFDocumentLoadingTask, TextItem and TextMarkedContent, all type-only imports that are erased at build. That check found a dead option at once: pdfjs 6 has no isEvalSupported. The option went away with the font-program eval path it used to switch off, so it did nothing here; it is removed. --- src/lib/documents/local-extract.ts | 45 ++++++++---------------------- 1 file changed, 11 insertions(+), 34 deletions(-) diff --git a/src/lib/documents/local-extract.ts b/src/lib/documents/local-extract.ts index 39b33df09..98afd9935 100644 --- a/src/lib/documents/local-extract.ts +++ b/src/lib/documents/local-extract.ts @@ -28,6 +28,12 @@ */ import { Buffer } from "node:buffer"; +import type { PDFDocumentLoadingTask } from "pdfjs-dist"; +import type { + TextItem, + TextMarkedContent, +} from "pdfjs-dist/types/src/display/api"; + import { annotate } from "@/lib/logging/context"; // pdfjs' module top-level references the browser-only `DOMMatrix` global. A @@ -35,42 +41,15 @@ import { annotate } from "@/lib/logging/context"; // instantiation and throw `DOMMatrix is not defined`, taking down every // route/worker that shares the chunk (the document routes among them). The // module is therefore loaded lazily inside `extractPdfText` via a runtime -// `import()`, and only the slice of its API used here is typed, structurally. -interface PdfTextItem { - str: string; - /** [a, b, c, d, x, y]: the run's text matrix; x and y place it on the page. */ - transform: number[]; - width: number; - height: number; - hasEOL: boolean; -} -interface PdfTextPage { - getTextContent(): Promise<{ items: Array }>; - cleanup(): void; -} -interface PdfTextDocument { - numPages: number; - getPage(pageNumber: number): Promise; -} -interface PdfTextLoadingTask { - promise: Promise; - destroy(): Promise; -} -interface PdfTextModule { - getDocument(opts: { - data: Uint8Array; - verbosity?: number; - isEvalSupported?: boolean; - useSystemFonts?: boolean; - }): PdfTextLoadingTask; -} +// `import()`. The type imports above are erased at build, so they load +// nothing; they let the compiler check this file against pdfjs' own API. /** * One page's text runs as text: a new line where a run ends its line or the * next one sits on another, a space where two runs on one line leave a gap, * nothing where they touch (a word split across two fonts stays one word). */ -function pageText(items: Array): string { +function pageText(items: Array): string { let text = ""; let last: { end: number; y: number; size: number } | null = null; for (const item of items) { @@ -128,16 +107,14 @@ export type LocalExtractResult = export async function extractPdfText( buffer: Buffer, ): Promise { - let task: PdfTextLoadingTask | null = null; + let task: PDFDocumentLoadingTask | null = null; try { // Lazy load: keeps pdfjs/DOMMatrix out of the eager server chunk (see the // import-site note). Resolved at first PDF extraction, then module-cached. - const pdfjs = - (await import("pdfjs-dist/legacy/build/pdf.mjs")) as unknown as PdfTextModule; + const pdfjs = await import("pdfjs-dist/legacy/build/pdf.mjs"); task = pdfjs.getDocument({ data: new Uint8Array(buffer), verbosity: 0, // VerbosityLevel.ERRORS — no console spam on odd PDFs. - isEvalSupported: false, // never eval font programs (defence in depth). useSystemFonts: false, }); const doc = await task.promise; From ce6c7528640a5bea0d84f1a1d829fa4df8f764ff Mon Sep 17 00:00:00 2001 From: haritowa <6107211+haritowa@users.noreply.github.com> Date: Fri, 23 Oct 2026 18:37:23 +0000 Subject: [PATCH 6/9] refactor(documents): name the steps of the page text joiner pageText worked on raw text-matrix slots and bare numbers. Each step now has a name: positionOf reads where a run starts and ends and its font size, startsNewLine says when the baseline moved by more than half a font size, and leavesWordGap says when the gap is wider than a tenth of the font size. The output is the same on every test PDF. --- src/lib/documents/local-extract.ts | 75 +++++++++++++++++++++++------- 1 file changed, 59 insertions(+), 16 deletions(-) diff --git a/src/lib/documents/local-extract.ts b/src/lib/documents/local-extract.ts index 98afd9935..e726f4a27 100644 --- a/src/lib/documents/local-extract.ts +++ b/src/lib/documents/local-extract.ts @@ -45,29 +45,72 @@ import { annotate } from "@/lib/logging/context"; // nothing; they let the compiler check this file against pdfjs' own API. /** - * One page's text runs as text: a new line where a run ends its line or the - * next one sits on another, a space where two runs on one line leave a gap, - * nothing where they touch (a word split across two fonts stays one word). + * Where a text run sits on the page. pdfjs gives each run a text matrix + * `[a, b, c, d, x, y]`: `x` and `y` are where the run starts, and `d` is its + * vertical scale, which for upright text is the font size. + */ +interface RunPosition { + x: number; + /** The baseline the run sits on. */ + y: number; + /** Where the run ends: its start plus its width. */ + end: number; + fontSize: number; +} + +function positionOf(run: TextItem): RunPosition { + const [, , , scaleY, x = 0, y = 0] = run.transform as number[]; + return { + x, + y, + end: x + run.width, + fontSize: Math.abs(scaleY ?? 0) || run.height || 1, + }; +} + +/** + * The next run is on a new line when its baseline moved by more than half a + * font size. Runs on one line can sit a little higher or lower (a superscript, + * a mixed font), so a small shift does not count. + */ +function startsNewLine(previous: RunPosition, next: RunPosition): boolean { + return Math.abs(next.y - previous.y) > previous.fontSize / 2; +} + +/** + * Two runs on one line are separate words when the gap between them is wider + * than a tenth of the font size. Runs that touch, such as a word drawn in two + * fonts, stay one word. + */ +function leavesWordGap(previous: RunPosition, next: RunPosition): boolean { + return next.x - previous.end > next.fontSize / 10; +} + +/** + * One page's text: its runs in order, with a line break where a run ends a + * line or the next one starts a new line, and a space where two runs on one + * line leave a gap between words. */ function pageText(items: Array): string { let text = ""; - let last: { end: number; y: number; size: number } | null = null; + let previous: RunPosition | null = null; + for (const item of items) { - // Marked-content boundaries carry no text. + // Marked-content markers carry no text. if (!("str" in item)) continue; - const x = item.transform[4] ?? 0; - const y = item.transform[5] ?? 0; - const size = Math.abs(item.transform[3] ?? 0) || item.height || 1; - if (last && !text.endsWith("\n")) { - if (Math.abs(y - last.y) > last.size / 2) text += "\n"; - else if (x - last.end > size * 0.1 && !/\s$/.test(text)) text += " "; + const position = positionOf(item); + + if (previous && !text.endsWith("\n")) { + if (startsNewLine(previous, position)) { + text += "\n"; + } else if (leavesWordGap(previous, position) && !/\s$/.test(text)) { + text += " "; + } } text += item.str; - // An empty run that only ends a line adds no blank line of its own. - if (item.hasEOL && !(item.str === "" && text.endsWith("\n"))) { - text += "\n"; - } - last = { end: x + item.width, y, size }; + if (item.hasEOL && !text.endsWith("\n")) text += "\n"; + + previous = position; } return text; } From 540c6bfaeb3045513a8cf93d5706c96b3f19ec6f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Sat, 24 Oct 2026 20:16:01 +0200 Subject: [PATCH 7/9] refactor(documents): read the text layer through the one shared loader The text read and the rasterizer each opened pdf.js on their own: the text read with bare options, the rasterizer through src/lib/documents/pdfjs.ts with the standard font, CMap and wasm directories. The loader now serves both, so a CID font with a predefined CMap is mapped to characters on the text path too, and there is one set of options to keep in step. The loader is typed with pdf.js' own declarations instead of a hand-written shape. That drops isEvalSupported, which pdf.js 6 no longer has and which did nothing. Only the render slice the DOM-typed declarations cannot describe (the native canvas factory) keeps a named local type in the rasterizer. The run joiner with its named steps moves into the loader as joinTextRuns, and readPdfText is its one page walk; the second layout copy that mimicked the previous library's tabs and blank lines is gone. extractPdfText keeps its outcomes and page cap on top of it, and a failed read still says what pdf.js threw (documents.local_extract.failed). A test draws a word in two fonts into a real PDF and reads it back whole. Refs #1277 --- .../__tests__/pdfjs-text-layer.test.ts | 118 +++++---- src/lib/documents/local-extract.ts | 154 +++--------- src/lib/documents/pdfjs.ts | 232 +++++++----------- src/lib/documents/rasterize-pdf.ts | 54 ++-- 4 files changed, 224 insertions(+), 334 deletions(-) diff --git a/src/lib/documents/__tests__/pdfjs-text-layer.test.ts b/src/lib/documents/__tests__/pdfjs-text-layer.test.ts index aae80e893..ad331d608 100644 --- a/src/lib/documents/__tests__/pdfjs-text-layer.test.ts +++ b/src/lib/documents/__tests__/pdfjs-text-layer.test.ts @@ -1,80 +1,89 @@ import { jsPDF } from "jspdf"; +import type { + TextItem, + TextMarkedContent, +} from "pdfjs-dist/types/src/display/api"; import { describe, expect, it } from "vitest"; -import { pageTextLayer, readPdfText, type PdfTextItem } from "../pdfjs"; +import { joinTextRuns, readPdfText } from "../pdfjs"; /** - * The text layer as the content index reads it. The layout is the one the - * local text read produced before it moved onto the app's own pdf.js (#1277): - * a break where pdf.js marks one or the next item sits on a lower line, a tab - * between far-apart items on one line, each page followed by a blank line. + * The text layer as the content index reads it (#1277): runs in order, a line + * break where a run ends its line or the next one sits on another, a space + * where two runs on one line leave a gap, and nothing where they touch, so a + * word drawn in two fonts stays one word and a search for it still finds it. */ -/** A page whose viewport maps PDF space to itself, flipped like pdf.js. */ -function pageOf(items: PdfTextItem[]) { - return { - getViewport: () => ({ - width: 600, - height: 800, - convertToViewportPoint: (x: number, y: number) => [x, 800 - y], - }), - getTextContent: async () => ({ items }), - }; -} - -function item( +/** A 10 pt run starting at (x, y), 5 pt per character. */ +function run( str: string, x: number, y: number, - extra: Partial = {}, -): PdfTextItem { + extra: Partial = {}, +): TextItem { return { str, + dir: "ltr", transform: [10, 0, 0, 10, x, y], width: str.length * 5, height: 10, + fontName: "g_d0_f1", hasEOL: false, ...extra, }; } -describe("pageTextLayer", () => { - it("breaks once where pdf.js marks the end of a line", async () => { - const text = await pageTextLayer( - pageOf([ - item("Haemoglobin", 10, 700, { hasEOL: true }), - item("14.2 g/dL", 10, 688), +describe("joinTextRuns", () => { + it("keeps a word drawn in two fonts whole", () => { + expect( + joinTextRuns([ + run("Haem", 10, 700), + run("oglobin", 30, 700, { fontName: "g_d0_f2" }), ]), - ); - expect(text).toBe("Haemoglobin\n14.2 g/dL"); + ).toBe("Haemoglobin"); }); - it("breaks when the next item sits a line lower without a marked end", async () => { - const text = await pageTextLayer( - pageOf([item("Creatinine", 10, 700), item("0.9 mg/dL", 10, 680)]), + it("puts a space between runs on one line that leave a gap", () => { + expect(joinTextRuns([run("Glucose", 10, 700), run("92", 200, 700)])).toBe( + "Glucose 92", ); - expect(text).toBe("Creatinine\n0.9 mg/dL"); }); - it("does not break for a small baseline shift on one line", async () => { - const text = await pageTextLayer( - pageOf([item("10", 10, 700), item("3", 21, 703)]), - ); - expect(text).toBe("103"); + it("breaks once where a run ends its line", () => { + expect( + joinTextRuns([ + run("Haemoglobin", 10, 700, { hasEOL: true }), + run("14.2 g/dL", 10, 688), + ]), + ).toBe("Haemoglobin\n14.2 g/dL"); }); - it("separates far-apart items on one line with a tab", async () => { - const text = await pageTextLayer( - pageOf([item("Glucose", 10, 700), item("92", 200, 700)]), - ); - expect(text).toBe("Glucose\t92"); + it("breaks where the next run sits on another line without a marked end", () => { + expect( + joinTextRuns([run("Creatinine", 10, 700), run("0.9", 10, 680)]), + ).toBe("Creatinine\n0.9"); }); - it("skips marked-content items, which carry no text", async () => { - const text = await pageTextLayer( - pageOf([{ hasEOL: false } as PdfTextItem, item("Text", 10, 700)]), - ); - expect(text).toBe("Text"); + it("keeps a raised run on its line", () => { + expect(joinTextRuns([run("10", 10, 700), run("9", 20, 703)])).toBe("109"); + }); + + it("adds no blank line for an empty run that only ends a line", () => { + expect( + joinTextRuns([ + run("Line", 10, 700, { hasEOL: true }), + run("", 30, 700, { hasEOL: true }), + run("Next", 10, 688), + ]), + ).toBe("Line\nNext"); + }); + + it("skips marked-content markers, which carry no text", () => { + const marker: TextMarkedContent = { + type: "beginMarkedContent", + id: "mc0", + }; + expect(joinTextRuns([marker, run("Text", 10, 700)])).toBe("Text"); }); }); @@ -89,10 +98,10 @@ describe("readPdfText", () => { return Buffer.from(doc.output("arraybuffer")); } - it("joins pages with a blank line and adds no page markers", async () => { + it("reads the pages one after another and adds no page markers", async () => { const text = await readPdfText(pages(2)); expect(text).toBe( - "Page 1 lab report\nsecond line\n\nPage 2 lab report\nsecond line\n\n", + "Page 1 lab report\nsecond line\nPage 2 lab report\nsecond line", ); }); @@ -102,6 +111,17 @@ describe("readPdfText", () => { expect(text).not.toContain("Page 3"); }); + it("keeps a word whole that a real PDF draws in two fonts", async () => { + const doc = new jsPDF(); + doc.setFont("helvetica", "bold"); + doc.text("Haem", 10, 10); + const joint = 10 + doc.getTextWidth("Haem"); + doc.setFont("times", "normal"); + doc.text("oglobin 14.2 g/dL", joint, 10); + const text = await readPdfText(Buffer.from(doc.output("arraybuffer"))); + expect(text).toBe("Haemoglobin 14.2 g/dL"); + }); + it("throws what pdf.js throws on a file that is not a PDF", async () => { await expect(readPdfText(Buffer.from("not a pdf"))).rejects.toThrow(); }); diff --git a/src/lib/documents/local-extract.ts b/src/lib/documents/local-extract.ts index e726f4a27..327f8a8b9 100644 --- a/src/lib/documents/local-extract.ts +++ b/src/lib/documents/local-extract.ts @@ -7,13 +7,14 @@ * never the native `@napi-rs/canvas`), in milliseconds, with no OCR and no * third-party egress. * - * It reads through the app's own `pdfjs-dist`, the copy `./rasterize-pdf` - * renders with. It used to go through `pdf-parse`, which brings a second - * pdfjs of its own, and the two cannot share a process: both take their - * worker from `globalThis.pdfjsWorker`, so whichever ran first broke the - * other ("The API version "6.4.299" does not match the Worker version - * "5.4.296""). Thumbnails and vision reads failed after a text extraction, - * and the text extraction failed after a raster. + * It reads through `./pdfjs`, the one loader `./rasterize-pdf` opens documents + * with too, so both run the app's own `pdfjs-dist` with the same options. It + * used to go through `pdf-parse`, which brings a second pdfjs of its own, and + * the two cannot share a process: both take their worker from + * `globalThis.pdfjsWorker`, so whichever ran first broke the other ("The API + * version "6.4.299" does not match the Worker version "5.4.296""). Thumbnails + * and vision reads failed after a text extraction, and the text extraction + * failed after a raster. * The extracted text feeds the SAME blind, encrypted content index the provider * path writes (`upsertContentIndex`), so nothing readable is stored at rest. * @@ -28,92 +29,10 @@ */ import { Buffer } from "node:buffer"; -import type { PDFDocumentLoadingTask } from "pdfjs-dist"; -import type { - TextItem, - TextMarkedContent, -} from "pdfjs-dist/types/src/display/api"; - import { annotate } from "@/lib/logging/context"; +import { logCaught } from "@/lib/logging/signal"; -// pdfjs' module top-level references the browser-only `DOMMatrix` global. A -// STATIC import makes the Turbopack server chunk evaluate that reference at -// instantiation and throw `DOMMatrix is not defined`, taking down every -// route/worker that shares the chunk (the document routes among them). The -// module is therefore loaded lazily inside `extractPdfText` via a runtime -// `import()`. The type imports above are erased at build, so they load -// nothing; they let the compiler check this file against pdfjs' own API. - -/** - * Where a text run sits on the page. pdfjs gives each run a text matrix - * `[a, b, c, d, x, y]`: `x` and `y` are where the run starts, and `d` is its - * vertical scale, which for upright text is the font size. - */ -interface RunPosition { - x: number; - /** The baseline the run sits on. */ - y: number; - /** Where the run ends: its start plus its width. */ - end: number; - fontSize: number; -} - -function positionOf(run: TextItem): RunPosition { - const [, , , scaleY, x = 0, y = 0] = run.transform as number[]; - return { - x, - y, - end: x + run.width, - fontSize: Math.abs(scaleY ?? 0) || run.height || 1, - }; -} - -/** - * The next run is on a new line when its baseline moved by more than half a - * font size. Runs on one line can sit a little higher or lower (a superscript, - * a mixed font), so a small shift does not count. - */ -function startsNewLine(previous: RunPosition, next: RunPosition): boolean { - return Math.abs(next.y - previous.y) > previous.fontSize / 2; -} - -/** - * Two runs on one line are separate words when the gap between them is wider - * than a tenth of the font size. Runs that touch, such as a word drawn in two - * fonts, stay one word. - */ -function leavesWordGap(previous: RunPosition, next: RunPosition): boolean { - return next.x - previous.end > next.fontSize / 10; -} - -/** - * One page's text: its runs in order, with a line break where a run ends a - * line or the next one starts a new line, and a space where two runs on one - * line leave a gap between words. - */ -function pageText(items: Array): string { - let text = ""; - let previous: RunPosition | null = null; - - for (const item of items) { - // Marked-content markers carry no text. - if (!("str" in item)) continue; - const position = positionOf(item); - - if (previous && !text.endsWith("\n")) { - if (startsNewLine(previous, position)) { - text += "\n"; - } else if (leavesWordGap(previous, position) && !/\s$/.test(text)) { - text += " "; - } - } - text += item.str; - if (item.hasEOL && !text.endsWith("\n")) text += "\n"; - - previous = position; - } - return text; -} +import { readPdfText } from "./pdfjs"; /** * Minimum joined text length for a PDF to count as a real text-layer document. @@ -145,49 +64,32 @@ export type LocalExtractResult = * Extract a text-layer PDF's embedded text server-side. Returns `empty` for a * scanned/no-text-layer PDF and `error` for a malformed/encrypted one — never * throws, so a bad document can never abort the caller (the upload/backfill job - * just leaves it un-indexed). + * just leaves it un-indexed). An `error` leaves a `warn` line naming what + * pdf.js threw, because the outcome alone does not say why a document stayed + * out of the index. */ export async function extractPdfText( buffer: Buffer, ): Promise { - let task: PDFDocumentLoadingTask | null = null; + let text: string; try { - // Lazy load: keeps pdfjs/DOMMatrix out of the eager server chunk (see the - // import-site note). Resolved at first PDF extraction, then module-cached. - const pdfjs = await import("pdfjs-dist/legacy/build/pdf.mjs"); - task = pdfjs.getDocument({ - data: new Uint8Array(buffer), - verbosity: 0, // VerbosityLevel.ERRORS — no console spam on odd PDFs. - useSystemFonts: false, - }); - const doc = await task.promise; // Bound the page walk so a pathological many-page PDF costs no more. - const pageCount = Math.min(doc.numPages, LOCAL_MAX_PAGES); - const pages: string[] = []; - for (let n = 1; n <= pageCount; n++) { - const page = await doc.getPage(n); - try { - pages.push(pageText((await page.getTextContent()).items)); - } finally { - page.cleanup(); - } - } - const text = pages.join("\n").trim(); - if (text.length < LOCAL_TEXT_MIN_CHARS) { - return { ok: false, reason: "empty" }; - } - return { ok: true, text, source: "local-pdf" }; - } catch { + text = (await readPdfText(buffer, LOCAL_MAX_PAGES)).trim(); + } catch (err) { + const errorType = err instanceof Error ? err.name || "Error" : typeof err; + annotate({ + action: { name: "documents.localIndex.extractFailed" }, + meta: { localExtractError: errorType }, + }); + logCaught("documents.local_extract.failed", err, { + bytes: buffer.length, + }); return { ok: false, reason: "error" }; - } finally { - if (task) { - try { - await task.destroy(); - } catch { - // Best-effort teardown; a destroy failure must not mask the result. - } - } } + if (text.length < LOCAL_TEXT_MIN_CHARS) { + return { ok: false, reason: "empty" }; + } + return { ok: true, text, source: "local-pdf" }; } /** diff --git a/src/lib/documents/pdfjs.ts b/src/lib/documents/pdfjs.ts index 5d44c1f31..9409323e9 100644 --- a/src/lib/documents/pdfjs.ts +++ b/src/lib/documents/pdfjs.ts @@ -1,5 +1,6 @@ /** - * The one pdf.js this server runs, and the one way it is opened. + * The one pdf.js this server runs, the one way it is opened, and the one way + * its text layer is read. * * Under Node, pdf.js does not start a worker thread: it loads its worker code * into the same process and keeps it on `globalThis.pdfjsWorker`. Whichever @@ -15,7 +16,9 @@ * reads the browser-only `DOMMatrix` global, so a static import would make the * server chunk evaluate it at instantiation and throw `DOMMatrix is not * defined`, taking down every route and worker sharing the chunk. The import - * resolves once and is module-cached after that. + * resolves once and is module-cached after that. The type imports below are + * erased at build, so they load nothing; they let the compiler check every + * call here against pdf.js' own API. * * This module has no logging of its own, so the end-to-end suite can read * report text with it; the callers say what failed. @@ -25,74 +28,11 @@ import { existsSync } from "node:fs"; import { createRequire } from "node:module"; import { dirname, join } from "node:path"; -// Structural types for the slice of the pdf.js API used here and in the -// rasterizer. Type-only, so nothing evaluates pdf.js for its types. -export interface PdfViewport { - width: number; - height: number; - convertToViewportPoint(x: number, y: number): number[]; -} -export interface PdfCanvas { - width: number; - height: number; - toBuffer(mime: "image/jpeg", quality?: number): Buffer; -} -export interface PdfContext2D { - getImageData( - x: number, - y: number, - width: number, - height: number, - ): { data: Uint8ClampedArray }; -} -interface PdfCanvasAndContext { - canvas: PdfCanvas; - context: PdfContext2D; -} -interface PdfCanvasFactory { - create(width: number, height: number): PdfCanvasAndContext; - destroy?(target: PdfCanvasAndContext): void; -} -/** A text-layer item; marked-content items carry no `str` and are skipped. */ -export interface PdfTextItem { - str?: string; - hasEOL?: boolean; - transform?: number[]; - width?: number; - height?: number; -} -export interface PdfPage { - getViewport(opts: { scale: number }): PdfViewport; - render(opts: { - canvasContext: unknown; - viewport: PdfViewport; - canvas: unknown; - }): { promise: Promise }; - cleanup(): void; - getTextContent(): Promise<{ items: PdfTextItem[] }>; -} -export interface PdfDocument { - numPages: number; - canvasFactory: PdfCanvasFactory; - getPage(pageNumber: number): Promise; -} -export interface PdfLoadingTask { - promise: Promise; - destroy(): Promise; -} -interface PdfDocumentOptions { - data: Uint8Array; - verbosity?: number; - isEvalSupported?: boolean; - useSystemFonts?: boolean; - standardFontDataUrl?: string; - cMapUrl?: string; - cMapPacked?: boolean; - wasmUrl?: string; -} -interface PdfjsModule { - getDocument(opts: PdfDocumentOptions): PdfLoadingTask; -} +import type { PDFDocumentLoadingTask } from "pdfjs-dist"; +import type { + TextItem, + TextMarkedContent, +} from "pdfjs-dist/types/src/display/api"; /** * Where pdf.js finds the data it does not carry in its JS: the standard 14 @@ -169,15 +109,13 @@ export function __resetPdfjsAssetsForTests(): void { */ export async function openPdfDocument( data: Uint8Array | Buffer, -): Promise { - const pdfjs = - (await import("pdfjs-dist/legacy/build/pdf.mjs")) as unknown as PdfjsModule; +): Promise { + const pdfjs = await import("pdfjs-dist/legacy/build/pdf.mjs"); const assets = resolvePdfjsAssets(); return pdfjs.getDocument({ // A copy: pdf.js transfers the buffer it is given to its worker. data: new Uint8Array(data), verbosity: 0, // VerbosityLevel.ERRORS: no console noise on odd PDFs. - isEvalSupported: false, // never eval font programs (defence in depth). // There are no system fonts to use: substitutes come from pdf.js' own // standard font data (and, when rendering, the registered fallbacks). useSystemFonts: false, @@ -192,73 +130,83 @@ export async function openPdfDocument( }); } -/** Vertical gap (pt) past which the next item starts a new line. */ -const LINE_THRESHOLD = 4.6; -/** Horizontal gap (pt) on one line past which the next item is a new cell. */ -const CELL_THRESHOLD = 7; - /** - * One page's text layer as plain text: items in content order, a line break - * where pdf.js marks one or where the next item sits on a lower line, and a - * tab between items on one line that are far apart (table cells). - * - * This is the layout `pdf-parse` produced with its defaults, kept so that - * swapping it out did not change what the content index sees. + * Where a text run sits on the page. pdfjs gives each run a text matrix + * `[a, b, c, d, x, y]`: `x` and `y` are where the run starts, and `d` is its + * vertical scale, which for upright text is the font size. */ -export async function pageTextLayer( - page: Pick, -): Promise { - const viewport = page.getViewport({ scale: 1 }); - const { items } = await page.getTextContent(); - const parts: string[] = []; - let lastX: number | undefined; - let lastY: number | undefined; - let lineHeight = 0; - for (const item of items) { - if (typeof item.str !== "string") continue; - const transform = item.transform ?? [1, 0, 0, 1, 0, 0]; - const [x = 0, y = 0] = viewport.convertToViewportPoint( - transform[4] ?? 0, - transform[5] ?? 0, - ); - let str = item.str; - if (lastY !== undefined) { - const gap = Math.abs(lastY - y); - if (gap > LINE_THRESHOLD) { - const previous = parts.at(-1); - const opensLine = - str.startsWith("\n") || (str.trim() === "" && item.hasEOL === true); - if ( - previous !== undefined && - !previous.endsWith("\n") && - !opensLine && - gap - 1 > lineHeight - ) { - parts.push("\n"); - lineHeight = 0; - } - } else if ( - gap < LINE_THRESHOLD && - lastX !== undefined && - Math.abs(lastX - x) > CELL_THRESHOLD - ) { - str = `\t${str}`; - } - } - parts.push(str); - lastX = x + (item.width ?? 0); - lastY = y; - lineHeight = Math.max(lineHeight, item.height ?? 0); - if (item.hasEOL) parts.push("\n"); - if (item.hasEOL || str.endsWith("\n")) lineHeight = 0; - } - return parts.join(""); +interface RunPosition { + x: number; + /** The baseline the run sits on. */ + y: number; + /** Where the run ends: its start plus its width. */ + end: number; + fontSize: number; +} + +function positionOf(run: TextItem): RunPosition { + const [, , , scaleY, x = 0, y = 0] = run.transform as number[]; + return { + x, + y, + end: x + run.width, + fontSize: Math.abs(scaleY ?? 0) || run.height || 1, + }; } /** - * The text layer of the first `maxPages` pages, each page followed by a blank - * line. Throws what pdf.js throws (a malformed file, a password, a page that - * will not parse); the document is torn down either way. + * The next run is on a new line when its baseline moved by more than half a + * font size. Runs on one line can sit a little higher or lower (a superscript, + * a mixed font), so a small shift does not count. + */ +function startsNewLine(previous: RunPosition, next: RunPosition): boolean { + return Math.abs(next.y - previous.y) > previous.fontSize / 2; +} + +/** + * Two runs on one line are separate words when the gap between them is wider + * than a tenth of the font size. Runs that touch, such as a word drawn in two + * fonts, stay one word. + */ +function leavesWordGap(previous: RunPosition, next: RunPosition): boolean { + return next.x - previous.end > next.fontSize / 10; +} + +/** + * One page's text: its runs in order, with a line break where a run ends a + * line or the next one starts a new line, and a space where two runs on one + * line leave a gap between words. + */ +export function joinTextRuns( + items: Array, +): string { + let text = ""; + let previous: RunPosition | null = null; + + for (const item of items) { + // Marked-content markers carry no text. + if (!("str" in item)) continue; + const position = positionOf(item); + + if (previous && !text.endsWith("\n")) { + if (startsNewLine(previous, position)) { + text += "\n"; + } else if (leavesWordGap(previous, position) && !/\s$/.test(text)) { + text += " "; + } + } + text += item.str; + if (item.hasEOL && !text.endsWith("\n")) text += "\n"; + + previous = position; + } + return text; +} + +/** + * The text layer of the first `maxPages` pages, one page after another. + * Throws what pdf.js throws (a malformed file, a password, a page that will + * not parse); the document is torn down either way. */ export async function readPdfText( data: Uint8Array | Buffer, @@ -267,17 +215,17 @@ export async function readPdfText( const task = await openPdfDocument(data); try { const doc = await task.promise; - const last = Math.min(doc.numPages, maxPages); - let text = ""; - for (let pageNumber = 1; pageNumber <= last; pageNumber++) { - const page = await doc.getPage(pageNumber); + const pageCount = Math.min(doc.numPages, maxPages); + const pages: string[] = []; + for (let n = 1; n <= pageCount; n++) { + const page = await doc.getPage(n); try { - text += `${await pageTextLayer(page)}\n\n`; + pages.push(joinTextRuns((await page.getTextContent()).items)); } finally { page.cleanup(); } } - return text; + return pages.join("\n"); } finally { try { await task.destroy(); diff --git a/src/lib/documents/rasterize-pdf.ts b/src/lib/documents/rasterize-pdf.ts index f23730e14..5bb98f32a 100644 --- a/src/lib/documents/rasterize-pdf.ts +++ b/src/lib/documents/rasterize-pdf.ts @@ -33,15 +33,7 @@ import { Buffer } from "node:buffer"; import { annotate } from "@/lib/logging/context"; import { nativeCanvasSupported } from "./native-canvas-support"; -import { - openPdfDocument, - resolvePdfjsAssets, - type PdfContext2D, - type PdfDocument, - type PdfjsAssets, - type PdfLoadingTask, - type PdfPage, -} from "./pdfjs"; +import { openPdfDocument, resolvePdfjsAssets, type PdfjsAssets } from "./pdfjs"; /** * Whether the PDF rasterizer is available in this build. The native @@ -114,6 +106,33 @@ export type RasterResult = } | { ok: false; reason: "unsupported" | "render-failed" }; +/** + * The canvas pdf.js draws into under Node is `@napi-rs/canvas`. pdf.js' own + * declarations are written for the DOM and type the document's canvas factory + * as a bare `object`, so the slice used here is named: a DOM-shaped canvas + * and 2D context, plus the JPEG encoder the native canvas adds. + */ +interface RenderTarget { + canvas: HTMLCanvasElement & { + toBuffer(mime: "image/jpeg", quality?: number): Buffer; + }; + context: CanvasRenderingContext2D; +} +interface RenderTargetFactory { + create(width: number, height: number): RenderTarget; + destroy?(target: RenderTarget): void; +} + +/** + * A page's text layer, as far as `pageText` reads it: text runs, and + * marked-content markers that carry only a `type` and no text. + */ +interface TextLayerPage { + getTextContent(): Promise<{ + items: Array<{ str?: string; hasEOL?: boolean; type?: string }>; + }>; +} + interface GlobalFontsLike { registerFromPath(path: string, nameAlias?: string): unknown; } @@ -144,7 +163,11 @@ async function registerFallbackFamilies(assets: PdfjsAssets): Promise { } /** Share of dark pixels on a rendered page (luminance below half). */ -function inkRatio(context: PdfContext2D, width: number, height: number) { +function inkRatio( + context: Pick, + width: number, + height: number, +) { const { data } = context.getImageData(0, 0, width, height); let dark = 0; for (let i = 0; i < data.length; i += 4) { @@ -188,9 +211,7 @@ function headThatFits(text: string, room: number): string { * `RASTER_PAGE_TEXT_MAX` it is cut (at a line, else a space, else hard) and * marked. */ -export async function pageText( - page: Pick, -): Promise { +export async function pageText(page: TextLayerPage): Promise { const max = RASTER_PAGE_TEXT_MAX; try { const { items } = await page.getTextContent(); @@ -248,8 +269,7 @@ export async function rasterizePdf( maxPages: number = RASTER_MAX_PAGES, opts: { withText?: boolean } = {}, ): Promise { - let doc: PdfDocument | null = null; - let task: PdfLoadingTask | null = null; + let task: Awaited> | null = null; // CPU-feature gate: pdfjs renders through @napi-rs/canvas (Skia), whose x64 // build uses AVX2 — an unsupported CPU dies with an uncatchable SIGILL on // the first render. Degrade to the existing no-raster path instead (scanned @@ -273,7 +293,7 @@ export async function rasterizePdf( } task = await openPdfDocument(buffer); - doc = await task.promise; + const doc = await task.promise; // Never exceed the standing DoS ceiling, even if a caller asks for more. const cap = Math.min(Math.max(1, maxPages), RASTER_HARD_MAX_PAGES); @@ -292,7 +312,7 @@ export async function rasterizePdf( longEdge > 0 ? Math.min(1, RASTER_TARGET_LONG_EDGE / longEdge) : 1; const viewport = page.getViewport({ scale }); - const factory = doc.canvasFactory; + const factory = doc.canvasFactory as RenderTargetFactory; const cc = factory.create( Math.ceil(viewport.width), Math.ceil(viewport.height), From 1fd34e4d749f6642abbc6c4bb8a95b26a88d0597 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Sat, 24 Oct 2026 20:16:02 +0200 Subject: [PATCH 8/9] test(documents): keep one pair of one-process order tests Two pairs of tests ran a text read and a rasterization in one process, one file per order. The pdfjs-one-version pair stays; the other pair and its shared helper go. The kept pair now first asserts that nothing in the process has claimed the pdf.js worker yet, so a file that lands in a shared process fails instead of passing on an order it never ran, and it mocks the failure signal the text read can emit. Refs #1277 --- .../pdfjs-one-runtime-raster-first.test.ts | 26 ---------- .../pdfjs-one-runtime-text-first.test.ts | 26 ---------- .../documents/__tests__/pdfjs-one-runtime.ts | 50 ------------------- .../pdfjs-one-version-extract-first.test.ts | 7 +++ .../pdfjs-one-version-raster-first.test.ts | 7 +++ 5 files changed, 14 insertions(+), 102 deletions(-) delete mode 100644 src/lib/documents/__tests__/pdfjs-one-runtime-raster-first.test.ts delete mode 100644 src/lib/documents/__tests__/pdfjs-one-runtime-text-first.test.ts delete mode 100644 src/lib/documents/__tests__/pdfjs-one-runtime.ts diff --git a/src/lib/documents/__tests__/pdfjs-one-runtime-raster-first.test.ts b/src/lib/documents/__tests__/pdfjs-one-runtime-raster-first.test.ts deleted file mode 100644 index 1af8dc44f..000000000 --- a/src/lib/documents/__tests__/pdfjs-one-runtime-raster-first.test.ts +++ /dev/null @@ -1,26 +0,0 @@ -import { describe, it, vi } from "vitest"; - -/** - * GitHub #1277, rasterization first: a render used to claim the in-process - * pdf.js worker for its own copy, and every later local text read failed - * without a word, leaving the document out of the search index. See - * `pdfjs-one-runtime.ts`. - */ - -vi.mock("@/lib/logging/context", () => ({ annotate: vi.fn() })); -vi.mock("@/lib/logging/signal", () => ({ logCaught: vi.fn() })); - -import { - expectFreshProcess, - expectRaster, - expectTextRead, -} from "./pdfjs-one-runtime"; - -describe("one pdf.js per process (#1277): rasterization first", () => { - it("reads text after a rasterization, and rasterizes again after that", async () => { - expectFreshProcess(); - await expectRaster(); - await expectTextRead(); - await expectRaster(); - }); -}); diff --git a/src/lib/documents/__tests__/pdfjs-one-runtime-text-first.test.ts b/src/lib/documents/__tests__/pdfjs-one-runtime-text-first.test.ts deleted file mode 100644 index 846ec1852..000000000 --- a/src/lib/documents/__tests__/pdfjs-one-runtime-text-first.test.ts +++ /dev/null @@ -1,26 +0,0 @@ -import { describe, it, vi } from "vitest"; - -/** - * GitHub #1277, text first: a local text read used to claim the in-process - * pdf.js worker for its own copy, and every later rasterization (thumbnails, - * the vision read of a scan) failed until a restart. See - * `pdfjs-one-runtime.ts`. - */ - -vi.mock("@/lib/logging/context", () => ({ annotate: vi.fn() })); -vi.mock("@/lib/logging/signal", () => ({ logCaught: vi.fn() })); - -import { - expectFreshProcess, - expectRaster, - expectTextRead, -} from "./pdfjs-one-runtime"; - -describe("one pdf.js per process (#1277): text read first", () => { - it("rasterizes after a text read, and reads text again after that", async () => { - expectFreshProcess(); - await expectTextRead(); - await expectRaster(); - await expectTextRead(); - }); -}); diff --git a/src/lib/documents/__tests__/pdfjs-one-runtime.ts b/src/lib/documents/__tests__/pdfjs-one-runtime.ts deleted file mode 100644 index d3fb7a672..000000000 --- a/src/lib/documents/__tests__/pdfjs-one-runtime.ts +++ /dev/null @@ -1,50 +0,0 @@ -/** - * Shared steps for the two `pdfjs-one-runtime-*` files (GitHub #1277). - * - * Under Node, pdf.js runs its worker in-process and keeps it on - * `globalThis.pdfjsWorker`. Whichever pdf.js copy opens a document first - * owns that global for the life of the process, and a different copy then - * refuses every document with "The API version … does not match the Worker - * version …". The text reader and the rasterizer once loaded different - * copies, so whichever ran first broke the other until a restart. - * - * Each order needs a fresh process, because the global cannot be taken back - * once claimed; that is why the two orders live in two files (the isolated - * project gives each file its own process) instead of two tests here. - */ -import { readFileSync } from "node:fs"; -import { join } from "node:path"; - -import { expect } from "vitest"; - -import { extractPdfText } from "../local-extract"; -import { rasterizePdf } from "../rasterize-pdf"; - -const FIXTURE = join( - process.cwd(), - "tests", - "fixtures", - "pdf", - "fallback-fonts.pdf", -); - -const pdf = () => readFileSync(FIXTURE); - -/** Nothing in this process has opened a PDF yet. */ -export function expectFreshProcess(): void { - expect( - (globalThis as { pdfjsWorker?: unknown }).pdfjsWorker, - "the file must run in its own process; another file already opened a PDF", - ).toBeUndefined(); -} - -export async function expectTextRead(): Promise { - const result = await extractPdfText(pdf()); - expect(result).toMatchObject({ ok: true, source: "local-pdf" }); - if (result.ok) expect(result.text).toContain("Test Insurance AG"); -} - -export async function expectRaster(): Promise { - const result = await rasterizePdf(pdf(), 1); - expect(result).toMatchObject({ ok: true }); -} diff --git a/src/lib/documents/__tests__/pdfjs-one-version-extract-first.test.ts b/src/lib/documents/__tests__/pdfjs-one-version-extract-first.test.ts index 298696510..596523e00 100644 --- a/src/lib/documents/__tests__/pdfjs-one-version-extract-first.test.ts +++ b/src/lib/documents/__tests__/pdfjs-one-version-extract-first.test.ts @@ -21,6 +21,7 @@ import { describe, expect, it, vi } from "vitest"; * `extractPdfText` → the raster step here fails. */ vi.mock("@/lib/logging/context", () => ({ annotate: vi.fn() })); +vi.mock("@/lib/logging/signal", () => ({ logCaught: vi.fn() })); import { extractPdfText } from "../local-extract"; import { rasterizePdf } from "../rasterize-pdf"; @@ -31,6 +32,12 @@ const fixture = readFileSync( describe("one pdfjs for text and raster: text first", () => { it("extracts, rasterises and extracts again in one process", async () => { + // The order only means something in a process that has not opened a PDF. + expect( + (globalThis as { pdfjsWorker?: unknown }).pdfjsWorker, + "this file must run in a process of its own", + ).toBeUndefined(); + const first = await extractPdfText(fixture); expect(first.ok).toBe(true); diff --git a/src/lib/documents/__tests__/pdfjs-one-version-raster-first.test.ts b/src/lib/documents/__tests__/pdfjs-one-version-raster-first.test.ts index 3389f078e..94b987be7 100644 --- a/src/lib/documents/__tests__/pdfjs-one-version-raster-first.test.ts +++ b/src/lib/documents/__tests__/pdfjs-one-version-raster-first.test.ts @@ -21,6 +21,7 @@ import { describe, expect, it, vi } from "vitest"; * `extractPdfText` → the text step here fails. */ vi.mock("@/lib/logging/context", () => ({ annotate: vi.fn() })); +vi.mock("@/lib/logging/signal", () => ({ logCaught: vi.fn() })); import { extractPdfText } from "../local-extract"; import { rasterizePdf } from "../rasterize-pdf"; @@ -31,6 +32,12 @@ const fixture = readFileSync( describe("one pdfjs for text and raster: raster first", () => { it("rasterises, extracts and rasterises again in one process", async () => { + // The order only means something in a process that has not opened a PDF. + expect( + (globalThis as { pdfjsWorker?: unknown }).pdfjsWorker, + "this file must run in a process of its own", + ).toBeUndefined(); + const first = await rasterizePdf(fixture, 1); expect(first.ok).toBe(true); From b716ee590bed99e58bc198cd097ec39b06915db6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Sat, 24 Oct 2026 20:16:07 +0200 Subject: [PATCH 9/9] test: fold the two one-pdf.js guards into one Two guards with the same file name froze the same failure from two sides: one stopped a runtime import of pdf-parse, the other pinned one pdfjs-dist in package.json, the lockfile and the checkout, and allowed only the shared loader to import it. They are one guard now. It strips comments before it matches, so prose naming a package does not count, walks every runtime extension and skips __mocks__ as well as __tests__, treats a type-only import of pdfjs-dist outside the loader as an import, and keeps the import shapes each side's self-test proved, including the near-miss name that must not match. Refs #1277 --- src/__tests__/single-pdfjs-guard.test.ts | 228 +++++++++++++++++++---- 1 file changed, 187 insertions(+), 41 deletions(-) diff --git a/src/__tests__/single-pdfjs-guard.test.ts b/src/__tests__/single-pdfjs-guard.test.ts index d42270995..6f7d67b77 100644 --- a/src/__tests__/single-pdfjs-guard.test.ts +++ b/src/__tests__/single-pdfjs-guard.test.ts @@ -1,67 +1,196 @@ /** - * Structural guard: the server runs one pdfjs. + * Structural guard: the server runs one pdf.js, and only one module opens it. * - * ## The failure this freezes + * ## The failure this freezes (GitHub #1277) * - * `pdf-parse` brings its own `pdfjs-dist` beside the app's, and under Node - * every pdfjs copy keeps its worker on the one `globalThis.pdfjsWorker`. While - * local text extraction went through pdf-parse, whichever copy ran first in a - * process broke the other: "The API version "6.4.299" does not match the - * Worker version "5.4.296"". Thumbnails and vision reads failed after a text - * extraction, and the text extraction failed after a raster. + * Under Node, pdf.js keeps its in-process worker on `globalThis.pdfjsWorker`, + * and whichever copy opens a document first owns it until the process exits. + * Local text extraction went through `pdf-parse`, which pinned its own, older + * pdfjs-dist, while the rasterizer ran the app's. Whichever ran first broke + * the other with "The API version "6.4.299" does not match the Worker version + * "5.4.296"": after a text extraction every thumbnail and every vision read + * of a scan failed, after a raster every local text extraction failed without + * a word, until a restart. Nothing in the gate noticed, because each path + * passed on its own and the image tracing shipped every `pdfjs-dist@*` the + * store held. * * ## What it proves and what it does not * - * No runtime file under `src/` imports `pdf-parse`, so the only pdfjs the - * server loads is the app's own. Tests may still read generated PDFs back with - * it; they run in their own processes. The pdfjs-one-version tests prove the - * two orders in one process; this file stops the import from coming back. + * - package.json pins pdfjs-dist exactly and lists no pdf-parse. + * - The lockfile resolves exactly one pdfjs-dist, the version package.json + * pins, and no package in it depends on pdfjs-dist except the app itself. + * A new dependency that brings its own pdf.js, even the same version, + * fails here and has to be looked at; every line of the lockfile that + * names pdfjs-dist is either understood or reported. + * - The checkout resolves that same version from the working directory, + * which is how the server finds it. + * - No runtime module under `src/` imports pdf-parse, and none imports + * pdfjs-dist (a type-only import included) except + * `src/lib/documents/pdfjs.ts`, so the text read and the rasterizer open + * documents with the same options. Comments are stripped first, so prose + * that names a package does not count. + * + * It does not prove the image carries one copy; the Dockerfile hoists the + * pinned version by name and the tracing glob can only find what the lockfile + * installed. The behaviour itself is pinned by the two `pdfjs-one-version-*` + * tests under `src/lib/documents/__tests__/`, one per order, each in a + * process of its own. */ +import { createRequire } from "node:module"; import { join } from "node:path"; import { describe, expect, it } from "vitest"; import { + readSourceFolder, readSourceText, - requireFloor, stripComments, - walkSourceFiles, } from "./helpers/source-files"; -const SRC = join(process.cwd(), "src"); +const ROOT = process.cwd(); +/** Relative to `src/`. */ +const LOADER = "lib/documents/pdfjs.ts"; -/** - * Every runtime module under `src/`: the suites, mocks and generated client - * are not loaded by the server. About 3,400 today; the floor sits below that - * with headroom, so a walk that comes back short fails instead of passing. - */ -function runtimeFiles(): string[] { - const files = walkSourceFiles(SRC, { - floor: 5000, - extensions: [".ts", ".tsx", ".js", ".jsx", ".mjs", ".cjs", ".mts", ".cts"], - }).filter( - (rel) => - !rel.startsWith("generated/") && - !rel.split("/").includes("__tests__") && - !rel.split("/").includes("__mocks__") && - !/\.test\.(ts|tsx)$/.test(rel), - ); - return [...requireFloor("runtime files under src/", files, 3000)]; +const pkg = JSON.parse(readSourceText(join(ROOT, "package.json"))) as { + dependencies?: Record; + devDependencies?: Record; +}; +const pinned = + pkg.dependencies?.["pdfjs-dist"] ?? pkg.devDependencies?.["pdfjs-dist"]; + +/** Every lockfile line naming pdfjs-dist, sorted into what it means. */ +function readLockfile(lockfile: string): { + versions: Set; + rootImporters: number; + unexplained: string[]; +} { + const lines = lockfile.split("\n"); + const versions = new Set(); + let rootImporters = 0; + const unexplained: string[] = []; + lines.forEach((line, i) => { + if (!line.includes("pdfjs-dist")) return; + // A resolved copy: ` pdfjs-dist@6.4.299:` in `packages:` and again in + // `snapshots:` (where a peer suffix in parentheses may follow). + const resolved = + /^ {2}'?pdfjs-dist@(\d+\.\d+\.\d+[\w.+-]*)(?:\(.*\))?'?:/.exec(line); + if (resolved) { + versions.add(resolved[1]!); + return; + } + // The app's own entry under `importers:`, followed by its specifier. + if ( + /^ {6}pdfjs-dist:\s*$/.test(line) && + /^\s+specifier:/.test(lines[i + 1] ?? "") + ) { + rootImporters++; + return; + } + // Anything else is another package depending on pdfjs-dist, an + // override, or a spelling this reader does not know: all reported. + unexplained.push(`${i + 1}: ${line.trim()}`); + }); + return { versions, rootImporters, unexplained }; } -/** A static import, a re-export, a dynamic import or a require of pdf-parse. */ -const PDF_PARSE_IMPORT = - /(?:\bfrom\s*|\bimport\s*\(\s*|\brequire\s*\(\s*|\bimport\s+)["']pdf-parse(?:\/[^"']*)?["']/; +/** A static import, a re-export, a dynamic import or a require of `name`. */ +function importOf(name: string): RegExp { + return new RegExp( + String.raw`(?:\bfrom\s*|\bimport\s*\(\s*|\brequire\s*\(\s*|\bimport\s+)["']` + + name + + String.raw`(?:/[^"']*)?["']`, + ); +} +const PDFJS_IMPORT = importOf("pdfjs-dist"); +const PDF_PARSE_IMPORT = importOf("pdf-parse"); -describe("one pdfjs in the server", () => { - it("no runtime file imports pdf-parse", () => { - const importers = runtimeFiles().filter((rel) => - PDF_PARSE_IMPORT.test(stripComments(readSourceText(join(SRC, rel)))), +/** The modules whose comment-free source matches `pattern`, sorted. */ +function importers(files: Map, pattern: RegExp): string[] { + return [...files] + .filter(([, source]) => pattern.test(source)) + .map(([path]) => path) + .sort(); +} + +/** + * Runtime modules under `src/`, keyed by their path relative to it, with + * comments stripped. The suites, mocks and generated client are not loaded + * by the server. + */ +function runtimeSources(): Map { + return new Map( + readSourceFolder("src", { + floor: 1000, + extensions: [ + ".ts", + ".tsx", + ".mts", + ".cts", + ".js", + ".jsx", + ".mjs", + ".cjs", + ], + include: (rel) => + !rel.startsWith("generated/") && + !rel.split("/").includes("__tests__") && + !rel.split("/").includes("__mocks__") && + !/\.(?:test|spec)\.[mc]?[tj]sx?$/.test(rel), + }).map(([rel, source]) => [rel, stripComments(source)]), + ); +} + +describe("one pdf.js (#1277)", () => { + it("package.json pins pdfjs-dist exactly and lists no pdf-parse", () => { + expect(pinned, "pdfjs-dist is a direct dependency").toMatch( + /^\d+\.\d+\.\d+$/, ); - expect(importers).toEqual([]); + expect(pkg.dependencies?.["pdf-parse"]).toBeUndefined(); + expect(pkg.devDependencies?.["pdf-parse"]).toBeUndefined(); }); - it("recognises each way of importing it", () => { + it("the lockfile resolves one pdfjs-dist, and only the app depends on it", () => { + const lock = readLockfile(readSourceText(join(ROOT, "pnpm-lock.yaml"))); + expect(lock.unexplained, "lockfile lines naming pdfjs-dist").toEqual([]); + expect([...lock.versions]).toEqual([pinned]); + expect(lock.rootImporters).toBe(1); + }); + + it("the checkout resolves the pinned copy", () => { + const anchor = createRequire(join(ROOT, "package.json")); + const resolved = JSON.parse( + readSourceText(anchor.resolve("pdfjs-dist/package.json")), + ) as { version: string }; + expect(resolved.version).toBe(pinned); + }); + + it("no runtime module imports pdf-parse, and only the shared loader imports pdf.js", () => { + const sources = runtimeSources(); + expect(sources.has(LOADER), `${LOADER} exists`).toBe(true); + expect(importers(sources, PDF_PARSE_IMPORT)).toEqual([]); + expect(importers(sources, PDFJS_IMPORT)).toEqual([LOADER]); + }); + + it("the readers catch what they exist to catch", () => { + const twoCopies = [ + "importers:", + " .:", + " devDependencies:", + " pdfjs-dist:", + " specifier: 6.4.299", + " version: 6.4.299", + "packages:", + " pdfjs-dist@5.4.296:", + " pdfjs-dist@6.4.299:", + "snapshots:", + " some-parser@2.4.5:", + " dependencies:", + " pdfjs-dist: 5.4.296", + ].join("\n"); + const lock = readLockfile(twoCopies); + expect([...lock.versions].sort()).toEqual(["5.4.296", "6.4.299"]); + expect(lock.unexplained).toEqual(["13: pdfjs-dist: 5.4.296"]); + for (const line of [ 'import { PDFParse } from "pdf-parse";', 'import type { PDFParse } from "pdf-parse";', @@ -72,5 +201,22 @@ describe("one pdfjs in the server", () => { expect(PDF_PARSE_IMPORT.test(line), line).toBe(true); } expect(PDF_PARSE_IMPORT.test('import x from "pdf-parser";')).toBe(false); + + const sources = new Map( + [ + [LOADER, 'await import("pdfjs-dist/legacy/build/pdf.mjs")'], + [ + "a", + 'import type { TextItem } from "pdfjs-dist/types/src/display/api";', + ], + ["b", "const m = await import(\n 'pdfjs-dist'\n);"], + ["c", 'import "pdfjs-dist/legacy/build/pdf.worker.mjs";'], + ["d", '// once read with import("pdfjs-dist") here'], + ["e", 'const name = "pdfjs-dist";'], + ].map(([rel, source]) => [rel!, stripComments(source!)]), + ); + expect(importers(sources, PDFJS_IMPORT)).toEqual( + ["a", "b", "c", LOADER].sort(), + ); }); });