From bb762135dff5711d6b38a7bd858f897b15ef16df Mon Sep 17 00:00:00 2001 From: haritowa <6107211+haritowa@users.noreply.github.com> Date: Wed, 21 Oct 2026 01:35:59 +0000 Subject: [PATCH 1/6] fix(documents): read a PDF's text with the app's own pdfjs Local text extraction went through pdf-parse, which pins its own pdfjs-dist (5.4.296) beside the app's (6.4.299), and both copies take their worker from globalThis.pdfjsWorker. Whichever ran first in a process broke the other: after a text extraction every raster failed with "The API version "6.4.299" does not match the Worker version "5.4.296"" (thumbnails raster-failed, vision reads of a PDF 422 extractFailed), and after a raster the text extraction failed and the local content index stayed empty without a trace. extractPdfText now reads the text layer with getTextContent on the pdfjs-dist the rasterizer uses, so only one pdfjs runs in the server. Pointing pdf-parse at 6.4 instead is not enough: its destroy() calls PDFDocumentProxy.destroy(), which pdfjs 6 no longer has, so every document would skip its teardown. Two tests run text and raster in one process, each order in its own file. --- .../pdfjs-one-version-extract-first.test.ts | 40 ++++++ .../pdfjs-one-version-raster-first.test.ts | 40 ++++++ src/lib/documents/content-index.ts | 2 +- src/lib/documents/index-document.ts | 2 +- src/lib/documents/local-extract.ts | 119 ++++++++++++++---- 5 files changed, 176 insertions(+), 27 deletions(-) create mode 100644 src/lib/documents/__tests__/pdfjs-one-version-extract-first.test.ts create mode 100644 src/lib/documents/__tests__/pdfjs-one-version-raster-first.test.ts diff --git a/src/lib/documents/__tests__/pdfjs-one-version-extract-first.test.ts b/src/lib/documents/__tests__/pdfjs-one-version-extract-first.test.ts new file mode 100644 index 000000000..3b20f39da --- /dev/null +++ b/src/lib/documents/__tests__/pdfjs-one-version-extract-first.test.ts @@ -0,0 +1,40 @@ +import { readFileSync } from "node:fs"; +import { join } from "node:path"; + +import { describe, expect, it, vi } from "vitest"; + +/** + * Text extraction and rasterising share one process, and pdfjs keeps its worker + * on `globalThis.pdfjsWorker`. While `pdf-parse` brought its own pdfjs + * (5.4.296) beside the app's (6.4.299), whichever ran first installed its + * worker and the other refused it: "The API version "6.4.299" does not match + * the Worker version "5.4.296"". Thumbnails failed as `raster-failed`, vision + * reads of a PDF answered 422 and the local text index stayed empty without a + * word. Reading the text through the app's own pdfjs is the fix; this file runs + * extraction first and `pdfjs-one-version-raster-first.test.ts` the reverse, + * each in a fresh process state. + * + * Mutation check: read the text through `pdf-parse` again in `extractPdfText` → + * the raster step here fails. + */ +vi.mock("@/lib/logging/context", () => ({ annotate: vi.fn() })); + +import { extractPdfText } from "../local-extract"; +import { rasterizePdf } from "../rasterize-pdf"; + +const fixture = readFileSync( + join(process.cwd(), "tests", "fixtures", "pdf", "fallback-fonts.pdf"), +); + +describe("one pdfjs for text and raster: text first", () => { + it("extracts, rasterises and extracts again in one process", async () => { + const first = await extractPdfText(fixture); + expect(first.ok).toBe(true); + + const raster = await rasterizePdf(fixture, 1); + expect(raster.ok).toBe(true); + + const again = await extractPdfText(fixture); + expect(again).toEqual(first); + }); +}); diff --git a/src/lib/documents/__tests__/pdfjs-one-version-raster-first.test.ts b/src/lib/documents/__tests__/pdfjs-one-version-raster-first.test.ts new file mode 100644 index 000000000..3986da2fe --- /dev/null +++ b/src/lib/documents/__tests__/pdfjs-one-version-raster-first.test.ts @@ -0,0 +1,40 @@ +import { readFileSync } from "node:fs"; +import { join } from "node:path"; + +import { describe, expect, it, vi } from "vitest"; + +/** + * Text extraction and rasterising share one process, and pdfjs keeps its worker + * on `globalThis.pdfjsWorker`. While `pdf-parse` brought its own pdfjs + * (5.4.296) beside the app's (6.4.299), whichever ran first installed its + * worker and the other refused it: "The API version "6.4.299" does not match + * the Worker version "5.4.296"". Thumbnails failed as `raster-failed`, vision + * reads of a PDF answered 422 and the local text index stayed empty without a + * word. Reading the text through the app's own pdfjs is the fix; the twin runs + * extraction first (`pdfjs-one-version-extract-first.test.ts`) and this file + * the reverse, each in a fresh process state. + * + * Mutation check: read the text through `pdf-parse` again in `extractPdfText` → + * the text step here fails. + */ +vi.mock("@/lib/logging/context", () => ({ annotate: vi.fn() })); + +import { extractPdfText } from "../local-extract"; +import { rasterizePdf } from "../rasterize-pdf"; + +const fixture = readFileSync( + join(process.cwd(), "tests", "fixtures", "pdf", "fallback-fonts.pdf"), +); + +describe("one pdfjs for text and raster: raster first", () => { + it("rasterises, extracts and rasterises again in one process", async () => { + const first = await rasterizePdf(fixture, 1); + expect(first.ok).toBe(true); + + const text = await extractPdfText(fixture); + expect(text.ok).toBe(true); + + const again = await rasterizePdf(fixture, 1); + expect(again.ok).toBe(true); + }); +}); diff --git a/src/lib/documents/content-index.ts b/src/lib/documents/content-index.ts index 4b88616f3..d9f789186 100644 --- a/src/lib/documents/content-index.ts +++ b/src/lib/documents/content-index.ts @@ -325,7 +325,7 @@ export async function loadDocumentChatText( * - `vision` → an AI provider transcribed the stored original (image or PDF, * incl. scanned) — the AI-first primary path. * - `text-ocr` → browser-OCR text posted by the client (opt-in local OCR). - * - `local-pdf` → server-side text-layer extraction of a PDF (`pdf-parse`), no + * - `local-pdf` → server-side text-layer extraction of a PDF (pdfjs), no * provider, no egress — the fallback when no provider is usable. * - `local-ocr` → server-side OCR of a scanned image/PDF (deferred follow-up). */ diff --git a/src/lib/documents/index-document.ts b/src/lib/documents/index-document.ts index 80c3ff25d..aef4348de 100644 --- a/src/lib/documents/index-document.ts +++ b/src/lib/documents/index-document.ts @@ -24,7 +24,7 @@ * configured, the toggle is OFF for an external pick, the provider can't * read this file (e.g. a text-layer PDF on a non-Anthropic account), the * daily budget is reached, or the provider call fails — fall back to - * server-side text-layer extraction (`pdf-parse`, no egress, milliseconds). + * server-side text-layer extraction (pdfjs, no egress, milliseconds). * So content search works at a baseline even with no AI, and every * text-layer PDF is searchable regardless of provider. * diff --git a/src/lib/documents/local-extract.ts b/src/lib/documents/local-extract.ts index d473bd043..8476eb803 100644 --- a/src/lib/documents/local-extract.ts +++ b/src/lib/documents/local-extract.ts @@ -3,9 +3,17 @@ * * When NO AI provider is configured — or the user has not consented to one — * content search still needs a baseline. This module extracts a PDF's embedded - * text layer SERVER-SIDE with `pdf-parse` (already a direct dependency; its - * `getText()` touches only the pure-JS/wasm pdfjs core, never the native - * `@napi-rs/canvas`), in milliseconds, with no OCR and no third-party egress. + * text layer SERVER-SIDE with pdfjs' `getTextContent` (the pure-JS/wasm core, + * never the native `@napi-rs/canvas`), in milliseconds, with no OCR and no + * third-party egress. + * + * It reads through the app's own `pdfjs-dist`, the copy `./rasterize-pdf` + * renders with. It used to go through `pdf-parse`, which brings a second + * pdfjs of its own, and the two cannot share a process: both take their + * worker from `globalThis.pdfjsWorker`, so whichever ran first broke the + * other ("The API version "6.4.299" does not match the Worker version + * "5.4.296""). Thumbnails and vision reads failed after a text extraction, + * and the text extraction failed after a raster. * The extracted text feeds the SAME blind, encrypted content index the provider * path writes (`upsertContentIndex`), so nothing readable is stored at rest. * @@ -20,19 +28,68 @@ */ import { Buffer } from "node:buffer"; -// `pdf-parse` pulls in pdfjs, whose module top-level references the browser-only -// `DOMMatrix` global. A STATIC import makes the Turbopack server chunk evaluate -// that reference at instantiation and throw `DOMMatrix is not defined`, taking -// down every route/worker that shares the chunk (the document routes among -// them). The value is therefore loaded lazily inside `extractPdfText` via a -// runtime `import()`, which lets pdf-parse initialise pdfjs on its own terms and -// never touches DOMMatrix at load — verified to extract a real text layer under -// plain Node. Keep the surface type as a type-only import (fully erased at -// build, no runtime module evaluation). -import type { PDFParse } from "pdf-parse"; - import { annotate } from "@/lib/logging/context"; +// pdfjs' module top-level references the browser-only `DOMMatrix` global. A +// STATIC import makes the Turbopack server chunk evaluate that reference at +// instantiation and throw `DOMMatrix is not defined`, taking down every +// route/worker that shares the chunk (the document routes among them). The +// module is therefore loaded lazily inside `extractPdfText` via a runtime +// `import()`, and only the slice of its API used here is typed, structurally. +interface PdfTextItem { + str: string; + /** [a, b, c, d, x, y]: the run's text matrix; x and y place it on the page. */ + transform: number[]; + width: number; + height: number; + hasEOL: boolean; +} +interface PdfTextPage { + getTextContent(): Promise<{ items: Array }>; + cleanup(): void; +} +interface PdfTextDocument { + numPages: number; + getPage(pageNumber: number): Promise; +} +interface PdfTextLoadingTask { + promise: Promise; + destroy(): Promise; +} +interface PdfTextModule { + getDocument(opts: { + data: Uint8Array; + verbosity?: number; + isEvalSupported?: boolean; + useSystemFonts?: boolean; + }): PdfTextLoadingTask; +} + +/** + * One page's text runs as text: a new line where a run ends its line or the + * next one sits on another, a space where two runs on one line leave a gap, + * nothing where they touch (a word split across two fonts stays one word). + */ +function pageText(items: Array): string { + let text = ""; + let last: { end: number; y: number; size: number } | null = null; + for (const item of items) { + // Marked-content boundaries carry no text. + if (!("str" in item)) continue; + const x = item.transform[4] ?? 0; + const y = item.transform[5] ?? 0; + const size = Math.abs(item.transform[3] ?? 0) || item.height || 1; + if (last && !text.endsWith("\n")) { + if (Math.abs(y - last.y) > last.size / 2) text += "\n"; + else if (x - last.end > size * 0.1 && !/\s$/.test(text)) text += " "; + } + text += item.str; + if (item.hasEOL) text += "\n"; + last = { end: x + item.width, y, size }; + } + return text; +} + /** * Minimum joined text length for a PDF to count as a real text-layer document. * Below this a PDF is treated as scanned/empty — a few stray glyphs from a form @@ -68,19 +125,31 @@ export type LocalExtractResult = export async function extractPdfText( buffer: Buffer, ): Promise { - let parser: PDFParse | null = null; + let task: PdfTextLoadingTask | null = null; try { // Lazy load: keeps pdfjs/DOMMatrix out of the eager server chunk (see the // import-site note). Resolved at first PDF extraction, then module-cached. - const { PDFParse } = await import("pdf-parse"); - parser = new PDFParse({ data: new Uint8Array(buffer) }); - const result = await parser.getText({ - // Bound the page walk and drop the default "-- N of M --" page markers so - // only real document text is tokenised. - first: LOCAL_MAX_PAGES, - pageJoiner: "", + const pdfjs = + (await import("pdfjs-dist/legacy/build/pdf.mjs")) as unknown as PdfTextModule; + task = pdfjs.getDocument({ + data: new Uint8Array(buffer), + verbosity: 0, // VerbosityLevel.ERRORS — no console spam on odd PDFs. + isEvalSupported: false, // never eval font programs (defence in depth). + useSystemFonts: false, }); - const text = (result.text ?? "").trim(); + const doc = await task.promise; + // Bound the page walk so a pathological many-page PDF costs no more. + const pageCount = Math.min(doc.numPages, LOCAL_MAX_PAGES); + const pages: string[] = []; + for (let n = 1; n <= pageCount; n++) { + const page = await doc.getPage(n); + try { + pages.push(pageText((await page.getTextContent()).items)); + } finally { + page.cleanup(); + } + } + const text = pages.join("\n").trim(); if (text.length < LOCAL_TEXT_MIN_CHARS) { return { ok: false, reason: "empty" }; } @@ -88,9 +157,9 @@ export async function extractPdfText( } catch { return { ok: false, reason: "error" }; } finally { - if (parser) { + if (task) { try { - await parser.destroy(); + await task.destroy(); } catch { // Best-effort teardown; a destroy failure must not mask the result. } From 275121391b3341c0994268e38d840b8355ad4b5a Mon Sep 17 00:00:00 2001 From: haritowa <6107211+haritowa@users.noreply.github.com> Date: Thu, 22 Oct 2026 16:48:16 +0000 Subject: [PATCH 2/6] test(documents): keep pdf-parse out of the server's runtime A structural guard fails when any runtime file under src/ imports pdf-parse again, which would bring its second pdfjs back into the server process. The next.config comment on the tracing includes now describes the text path as it is, and the extractor no longer turns an empty end-of-line run into a blank line of its own. --- next.config.ts | 13 ++-- src/__tests__/single-pdfjs-guard.test.ts | 78 ++++++++++++++++++++++++ src/lib/documents/local-extract.ts | 5 +- 3 files changed, 90 insertions(+), 6 deletions(-) create mode 100644 src/__tests__/single-pdfjs-guard.test.ts diff --git a/next.config.ts b/next.config.ts index 055b1dcf7..66662bb82 100644 --- a/next.config.ts +++ b/next.config.ts @@ -302,11 +302,14 @@ const nextConfig: NextConfig = { outputFileTracingIncludes: { "*": [ "./src/lib/ai/prompts/safety-contracts.*.yaml", - // Document AI local text extraction runs `pdf-parse` (→ pdfjs-dist) - // server-side in the content-index job — the provider-free fallback that - // keeps content search working with no AI configured. Both are pure JS + - // wasm; there is NO native `.node` on the text-layer path (`@napi-rs/canvas` - // is only pulled for image rendering, which `getText()` never invokes). + // Document AI local text extraction reads the text layer with pdfjs-dist + // (`getTextContent`, `src/lib/documents/local-extract.ts`) server-side in + // the content-index job — the provider-free fallback that keeps content + // search working with no AI configured. It is pure JS + wasm; there is NO + // native `.node` on the text-layer path (`@napi-rs/canvas` is only pulled + // for image rendering). Nothing at runtime imports `pdf-parse` any more + // (its own older pdfjs clashed with the app's); its glob below is dead + // weight kept until the image is rebuilt and checked without it. // The standalone tracer can miss pdfjs-dist's wasm decoders (they resolve // via `new URL(..., import.meta.url)`), so pin both packages into the // runtime image explicitly. pnpm keeps pdfjs-dist unhoisted under `.pnpm`. diff --git a/src/__tests__/single-pdfjs-guard.test.ts b/src/__tests__/single-pdfjs-guard.test.ts new file mode 100644 index 000000000..897189e1b --- /dev/null +++ b/src/__tests__/single-pdfjs-guard.test.ts @@ -0,0 +1,78 @@ +/** + * Structural guard: the server runs one pdfjs. + * + * ## The failure this freezes + * + * `pdf-parse` brings its own `pdfjs-dist` beside the app's, and under Node + * every pdfjs copy keeps its worker on the one `globalThis.pdfjsWorker`. While + * local text extraction went through pdf-parse, whichever copy ran first in a + * process broke the other: "The API version "6.4.299" does not match the + * Worker version "5.4.296"". Thumbnails and vision reads failed after a text + * extraction, and the text extraction failed after a raster. + * + * ## What it proves and what it does not + * + * No runtime file under `src/` imports `pdf-parse`, so the only pdfjs the + * server loads is the app's own. Tests may still read generated PDFs back with + * it; they run in their own processes. The pdfjs-one-version tests prove the + * two orders in one process; this file stops the import from coming back. + */ +import { readFileSync } from "node:fs"; +import { join } from "node:path"; + +import { describe, expect, it } from "vitest"; + +import { + requireFloor, + stripComments, + walkSourceFiles, +} from "./helpers/source-files"; + +const SRC = join(process.cwd(), "src"); + +/** + * Every runtime module under `src/`: the suites, mocks and generated client + * are not loaded by the server. About 3,400 today; the floor sits below that + * with headroom, so a walk that comes back short fails instead of passing. + */ +function runtimeFiles(): string[] { + const files = walkSourceFiles(SRC, { + floor: 5000, + extensions: [".ts", ".tsx", ".js", ".jsx", ".mjs", ".cjs", ".mts", ".cts"], + }).filter( + (rel) => + !rel.startsWith("generated/") && + !rel.split("/").includes("__tests__") && + !rel.split("/").includes("__mocks__") && + !/\.test\.(ts|tsx)$/.test(rel), + ); + return [...requireFloor("runtime files under src/", files, 3000)]; +} + +/** A static import, a re-export, a dynamic import or a require of pdf-parse. */ +const PDF_PARSE_IMPORT = + /(?:\bfrom\s*|\bimport\s*\(\s*|\brequire\s*\(\s*|\bimport\s+)["']pdf-parse(?:\/[^"']*)?["']/; + +describe("one pdfjs in the server", () => { + it("no runtime file imports pdf-parse", () => { + const importers = runtimeFiles().filter((rel) => + PDF_PARSE_IMPORT.test( + stripComments(readFileSync(join(SRC, rel), "utf8")), + ), + ); + expect(importers).toEqual([]); + }); + + it("recognises each way of importing it", () => { + for (const line of [ + 'import { PDFParse } from "pdf-parse";', + 'import type { PDFParse } from "pdf-parse";', + 'const { PDFParse } = await import("pdf-parse");', + "const m = require('pdf-parse/worker');", + 'export { PDFParse } from "pdf-parse";', + ]) { + expect(PDF_PARSE_IMPORT.test(line), line).toBe(true); + } + expect(PDF_PARSE_IMPORT.test('import x from "pdf-parser";')).toBe(false); + }); +}); diff --git a/src/lib/documents/local-extract.ts b/src/lib/documents/local-extract.ts index 8476eb803..39b33df09 100644 --- a/src/lib/documents/local-extract.ts +++ b/src/lib/documents/local-extract.ts @@ -84,7 +84,10 @@ function pageText(items: Array): string { else if (x - last.end > size * 0.1 && !/\s$/.test(text)) text += " "; } text += item.str; - if (item.hasEOL) text += "\n"; + // An empty run that only ends a line adds no blank line of its own. + if (item.hasEOL && !(item.str === "" && text.endsWith("\n"))) { + text += "\n"; + } last = { end: x + item.width, y, size }; } return text; From 5090a6fdc1ad41d9aaac5edeb6bcad4da4c52460 Mon Sep 17 00:00:00 2001 From: haritowa <6107211+haritowa@users.noreply.github.com> Date: Fri, 23 Oct 2026 17:31:25 +0000 Subject: [PATCH 3/6] chore(deps): say what the pdf-parse copy of pdfjs is still for The override comment said pdf-parse's pdfjs 5.4.296 drives local text extraction. Extraction now reads with the app's own pdfjs; only the tests still use pdf-parse, which has to stay on pdfjs 5 because its teardown calls an API pdfjs 6 removed. The range itself is unchanged. --- pnpm-workspace.yaml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index 2f95c35f1..aade691c4 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -73,8 +73,10 @@ overrides: # transitively, and `outputFileTracingIncludes` ships EVERY `pdfjs-dist@*` in # the store into the standalone image — so an unused transitive copy is still # a copy in the runtime image. The range is bounded on both ends deliberately: - # `pdf-parse` resolves 5.4.296, which predates the flaw and drives the local - # text-extraction path, so it must not be dragged onto a new major. + # `pdf-parse` resolves 5.4.296, which predates the flaw. Only the tests use it + # now (they read generated PDFs back); it must not be dragged onto a new + # major, because its teardown calls `PDFDocumentProxy.destroy()`, which pdfjs + # 6 no longer has. Local text extraction reads with the app's own pdfjs. "pdfjs-dist@>=5.6.83 <6.2.108": "6.2.108" # Deeply nested input drives DeepmergeTS into unbounded recursion until the # stack gives out. The copy here is Prisma's: `@prisma/config` still asks for From 85bef667dd0d9d5052da948877090fb752f1088d Mon Sep 17 00:00:00 2001 From: haritowa <6107211+haritowa@users.noreply.github.com> Date: Fri, 23 Oct 2026 17:56:38 +0000 Subject: [PATCH 4/6] test(documents): read files the way the realism ratchet expects The test-realism ratchet counts a test that calls readFileSync and names a source file. The guard now reads source through readSourceText, and the two one-process tests no longer name their twin file in a comment. Their doc comments are rewritten in plainer words. --- src/__tests__/single-pdfjs-guard.test.ts | 6 ++--- .../pdfjs-one-version-extract-first.test.ts | 25 +++++++++++-------- .../pdfjs-one-version-raster-first.test.ts | 25 +++++++++++-------- 3 files changed, 30 insertions(+), 26 deletions(-) diff --git a/src/__tests__/single-pdfjs-guard.test.ts b/src/__tests__/single-pdfjs-guard.test.ts index 897189e1b..d42270995 100644 --- a/src/__tests__/single-pdfjs-guard.test.ts +++ b/src/__tests__/single-pdfjs-guard.test.ts @@ -17,12 +17,12 @@ * it; they run in their own processes. The pdfjs-one-version tests prove the * two orders in one process; this file stops the import from coming back. */ -import { readFileSync } from "node:fs"; import { join } from "node:path"; import { describe, expect, it } from "vitest"; import { + readSourceText, requireFloor, stripComments, walkSourceFiles, @@ -56,9 +56,7 @@ const PDF_PARSE_IMPORT = describe("one pdfjs in the server", () => { it("no runtime file imports pdf-parse", () => { const importers = runtimeFiles().filter((rel) => - PDF_PARSE_IMPORT.test( - stripComments(readFileSync(join(SRC, rel), "utf8")), - ), + PDF_PARSE_IMPORT.test(stripComments(readSourceText(join(SRC, rel)))), ); expect(importers).toEqual([]); }); diff --git a/src/lib/documents/__tests__/pdfjs-one-version-extract-first.test.ts b/src/lib/documents/__tests__/pdfjs-one-version-extract-first.test.ts index 3b20f39da..298696510 100644 --- a/src/lib/documents/__tests__/pdfjs-one-version-extract-first.test.ts +++ b/src/lib/documents/__tests__/pdfjs-one-version-extract-first.test.ts @@ -4,18 +4,21 @@ import { join } from "node:path"; import { describe, expect, it, vi } from "vitest"; /** - * Text extraction and rasterising share one process, and pdfjs keeps its worker - * on `globalThis.pdfjsWorker`. While `pdf-parse` brought its own pdfjs - * (5.4.296) beside the app's (6.4.299), whichever ran first installed its - * worker and the other refused it: "The API version "6.4.299" does not match - * the Worker version "5.4.296"". Thumbnails failed as `raster-failed`, vision - * reads of a PDF answered 422 and the local text index stayed empty without a - * word. Reading the text through the app's own pdfjs is the fix; this file runs - * extraction first and `pdfjs-one-version-raster-first.test.ts` the reverse, - * each in a fresh process state. + * Text extraction and rasterising run in one process. In Node, pdfjs keeps its + * worker in one global, `globalThis.pdfjsWorker`. When `pdf-parse` brought its + * own pdfjs (5.4.296) beside the app's (6.4.299), the copy that ran first set + * that global, and the other copy refused it: "The API version "6.4.299" does + * not match the Worker version "5.4.296"". Thumbnails then failed as + * `raster-failed`, vision reads of a PDF answered 422, and the local text index + * stayed empty with no log line. The fix: read the text with the app's own + * pdfjs. * - * Mutation check: read the text through `pdf-parse` again in `extractPdfText` → - * the raster step here fails. + * This file extracts text first and rasterises second. Its twin in this + * folder runs the reverse order. Each order needs its own file, because a + * process keeps the first worker it loads. + * + * Mutation check: read the text through `pdf-parse` again in + * `extractPdfText` → the raster step here fails. */ vi.mock("@/lib/logging/context", () => ({ annotate: vi.fn() })); diff --git a/src/lib/documents/__tests__/pdfjs-one-version-raster-first.test.ts b/src/lib/documents/__tests__/pdfjs-one-version-raster-first.test.ts index 3986da2fe..3389f078e 100644 --- a/src/lib/documents/__tests__/pdfjs-one-version-raster-first.test.ts +++ b/src/lib/documents/__tests__/pdfjs-one-version-raster-first.test.ts @@ -4,18 +4,21 @@ import { join } from "node:path"; import { describe, expect, it, vi } from "vitest"; /** - * Text extraction and rasterising share one process, and pdfjs keeps its worker - * on `globalThis.pdfjsWorker`. While `pdf-parse` brought its own pdfjs - * (5.4.296) beside the app's (6.4.299), whichever ran first installed its - * worker and the other refused it: "The API version "6.4.299" does not match - * the Worker version "5.4.296"". Thumbnails failed as `raster-failed`, vision - * reads of a PDF answered 422 and the local text index stayed empty without a - * word. Reading the text through the app's own pdfjs is the fix; the twin runs - * extraction first (`pdfjs-one-version-extract-first.test.ts`) and this file - * the reverse, each in a fresh process state. + * Text extraction and rasterising run in one process. In Node, pdfjs keeps its + * worker in one global, `globalThis.pdfjsWorker`. When `pdf-parse` brought its + * own pdfjs (5.4.296) beside the app's (6.4.299), the copy that ran first set + * that global, and the other copy refused it: "The API version "6.4.299" does + * not match the Worker version "5.4.296"". Thumbnails then failed as + * `raster-failed`, vision reads of a PDF answered 422, and the local text index + * stayed empty with no log line. The fix: read the text with the app's own + * pdfjs. * - * Mutation check: read the text through `pdf-parse` again in `extractPdfText` → - * the text step here fails. + * This file rasterises first and extracts text second. Its twin in this + * folder runs the reverse order. Each order needs its own file, because a + * process keeps the first worker it loads. + * + * Mutation check: read the text through `pdf-parse` again in + * `extractPdfText` → the text step here fails. */ vi.mock("@/lib/logging/context", () => ({ annotate: vi.fn() })); From 2c5e9a80b9ee6ba43dde7cd4166addc573c0d535 Mon Sep 17 00:00:00 2001 From: haritowa <6107211+haritowa@users.noreply.github.com> Date: Fri, 23 Oct 2026 18:35:17 +0000 Subject: [PATCH 5/6] refactor(documents): type the text read with pdfjs' own declarations extractPdfText cast the dynamic import to a hand-written shape (`as unknown as`), so the compiler could not check the call against pdfjs. It now uses pdfjs' own types: PDFDocumentLoadingTask, TextItem and TextMarkedContent, all type-only imports that are erased at build. That check found a dead option at once: pdfjs 6 has no isEvalSupported. The option went away with the font-program eval path it used to switch off, so it did nothing here; it is removed. --- src/lib/documents/local-extract.ts | 45 ++++++++---------------------- 1 file changed, 11 insertions(+), 34 deletions(-) diff --git a/src/lib/documents/local-extract.ts b/src/lib/documents/local-extract.ts index 39b33df09..98afd9935 100644 --- a/src/lib/documents/local-extract.ts +++ b/src/lib/documents/local-extract.ts @@ -28,6 +28,12 @@ */ import { Buffer } from "node:buffer"; +import type { PDFDocumentLoadingTask } from "pdfjs-dist"; +import type { + TextItem, + TextMarkedContent, +} from "pdfjs-dist/types/src/display/api"; + import { annotate } from "@/lib/logging/context"; // pdfjs' module top-level references the browser-only `DOMMatrix` global. A @@ -35,42 +41,15 @@ import { annotate } from "@/lib/logging/context"; // instantiation and throw `DOMMatrix is not defined`, taking down every // route/worker that shares the chunk (the document routes among them). The // module is therefore loaded lazily inside `extractPdfText` via a runtime -// `import()`, and only the slice of its API used here is typed, structurally. -interface PdfTextItem { - str: string; - /** [a, b, c, d, x, y]: the run's text matrix; x and y place it on the page. */ - transform: number[]; - width: number; - height: number; - hasEOL: boolean; -} -interface PdfTextPage { - getTextContent(): Promise<{ items: Array }>; - cleanup(): void; -} -interface PdfTextDocument { - numPages: number; - getPage(pageNumber: number): Promise; -} -interface PdfTextLoadingTask { - promise: Promise; - destroy(): Promise; -} -interface PdfTextModule { - getDocument(opts: { - data: Uint8Array; - verbosity?: number; - isEvalSupported?: boolean; - useSystemFonts?: boolean; - }): PdfTextLoadingTask; -} +// `import()`. The type imports above are erased at build, so they load +// nothing; they let the compiler check this file against pdfjs' own API. /** * One page's text runs as text: a new line where a run ends its line or the * next one sits on another, a space where two runs on one line leave a gap, * nothing where they touch (a word split across two fonts stays one word). */ -function pageText(items: Array): string { +function pageText(items: Array): string { let text = ""; let last: { end: number; y: number; size: number } | null = null; for (const item of items) { @@ -128,16 +107,14 @@ export type LocalExtractResult = export async function extractPdfText( buffer: Buffer, ): Promise { - let task: PdfTextLoadingTask | null = null; + let task: PDFDocumentLoadingTask | null = null; try { // Lazy load: keeps pdfjs/DOMMatrix out of the eager server chunk (see the // import-site note). Resolved at first PDF extraction, then module-cached. - const pdfjs = - (await import("pdfjs-dist/legacy/build/pdf.mjs")) as unknown as PdfTextModule; + const pdfjs = await import("pdfjs-dist/legacy/build/pdf.mjs"); task = pdfjs.getDocument({ data: new Uint8Array(buffer), verbosity: 0, // VerbosityLevel.ERRORS — no console spam on odd PDFs. - isEvalSupported: false, // never eval font programs (defence in depth). useSystemFonts: false, }); const doc = await task.promise; From ce6c7528640a5bea0d84f1a1d829fa4df8f764ff Mon Sep 17 00:00:00 2001 From: haritowa <6107211+haritowa@users.noreply.github.com> Date: Fri, 23 Oct 2026 18:37:23 +0000 Subject: [PATCH 6/6] refactor(documents): name the steps of the page text joiner pageText worked on raw text-matrix slots and bare numbers. Each step now has a name: positionOf reads where a run starts and ends and its font size, startsNewLine says when the baseline moved by more than half a font size, and leavesWordGap says when the gap is wider than a tenth of the font size. The output is the same on every test PDF. --- src/lib/documents/local-extract.ts | 75 +++++++++++++++++++++++------- 1 file changed, 59 insertions(+), 16 deletions(-) diff --git a/src/lib/documents/local-extract.ts b/src/lib/documents/local-extract.ts index 98afd9935..e726f4a27 100644 --- a/src/lib/documents/local-extract.ts +++ b/src/lib/documents/local-extract.ts @@ -45,29 +45,72 @@ import { annotate } from "@/lib/logging/context"; // nothing; they let the compiler check this file against pdfjs' own API. /** - * One page's text runs as text: a new line where a run ends its line or the - * next one sits on another, a space where two runs on one line leave a gap, - * nothing where they touch (a word split across two fonts stays one word). + * Where a text run sits on the page. pdfjs gives each run a text matrix + * `[a, b, c, d, x, y]`: `x` and `y` are where the run starts, and `d` is its + * vertical scale, which for upright text is the font size. + */ +interface RunPosition { + x: number; + /** The baseline the run sits on. */ + y: number; + /** Where the run ends: its start plus its width. */ + end: number; + fontSize: number; +} + +function positionOf(run: TextItem): RunPosition { + const [, , , scaleY, x = 0, y = 0] = run.transform as number[]; + return { + x, + y, + end: x + run.width, + fontSize: Math.abs(scaleY ?? 0) || run.height || 1, + }; +} + +/** + * The next run is on a new line when its baseline moved by more than half a + * font size. Runs on one line can sit a little higher or lower (a superscript, + * a mixed font), so a small shift does not count. + */ +function startsNewLine(previous: RunPosition, next: RunPosition): boolean { + return Math.abs(next.y - previous.y) > previous.fontSize / 2; +} + +/** + * Two runs on one line are separate words when the gap between them is wider + * than a tenth of the font size. Runs that touch, such as a word drawn in two + * fonts, stay one word. + */ +function leavesWordGap(previous: RunPosition, next: RunPosition): boolean { + return next.x - previous.end > next.fontSize / 10; +} + +/** + * One page's text: its runs in order, with a line break where a run ends a + * line or the next one starts a new line, and a space where two runs on one + * line leave a gap between words. */ function pageText(items: Array): string { let text = ""; - let last: { end: number; y: number; size: number } | null = null; + let previous: RunPosition | null = null; + for (const item of items) { - // Marked-content boundaries carry no text. + // Marked-content markers carry no text. if (!("str" in item)) continue; - const x = item.transform[4] ?? 0; - const y = item.transform[5] ?? 0; - const size = Math.abs(item.transform[3] ?? 0) || item.height || 1; - if (last && !text.endsWith("\n")) { - if (Math.abs(y - last.y) > last.size / 2) text += "\n"; - else if (x - last.end > size * 0.1 && !/\s$/.test(text)) text += " "; + const position = positionOf(item); + + if (previous && !text.endsWith("\n")) { + if (startsNewLine(previous, position)) { + text += "\n"; + } else if (leavesWordGap(previous, position) && !/\s$/.test(text)) { + text += " "; + } } text += item.str; - // An empty run that only ends a line adds no blank line of its own. - if (item.hasEOL && !(item.str === "" && text.endsWith("\n"))) { - text += "\n"; - } - last = { end: x + item.width, y, size }; + if (item.hasEOL && !text.endsWith("\n")) text += "\n"; + + previous = position; } return text; }