diff --git a/src/__tests__/single-pdfjs-guard.test.ts b/src/__tests__/single-pdfjs-guard.test.ts index 38309e2b4..6f7d67b77 100644 --- a/src/__tests__/single-pdfjs-guard.test.ts +++ b/src/__tests__/single-pdfjs-guard.test.ts @@ -5,16 +5,18 @@ * * Under Node, pdf.js keeps its in-process worker on `globalThis.pdfjsWorker`, * and whichever copy opens a document first owns it until the process exits. - * The local text read ran a library that pinned its own, older pdfjs-dist, - * while the rasterizer ran the app's. Whichever ran first broke the other - * with "The API version … does not match the Worker version …": after a text - * read every thumbnail and every vision read of a scan failed, after a render - * every local text read failed without a word, until a restart. Nothing in - * the gate noticed, because each path passed on its own and the image tracing - * shipped every `pdfjs-dist@*` the store held. + * Local text extraction went through `pdf-parse`, which pinned its own, older + * pdfjs-dist, while the rasterizer ran the app's. Whichever ran first broke + * the other with "The API version "6.4.299" does not match the Worker version + * "5.4.296"": after a text extraction every thumbnail and every vision read + * of a scan failed, after a raster every local text extraction failed without + * a word, until a restart. Nothing in the gate noticed, because each path + * passed on its own and the image tracing shipped every `pdfjs-dist@*` the + * store held. * * ## What it proves and what it does not * + * - package.json pins pdfjs-dist exactly and lists no pdf-parse. * - The lockfile resolves exactly one pdfjs-dist, the version package.json * pins, and no package in it depends on pdfjs-dist except the app itself. * A new dependency that brings its own pdf.js, even the same version, @@ -22,21 +24,28 @@ * names pdfjs-dist is either understood or reported. * - The checkout resolves that same version from the working directory, * which is how the server finds it. - * - No production module under `src/` imports pdfjs-dist except - * `src/lib/documents/pdfjs.ts`, so both paths open documents with the same - * options, and nothing imports pdf-parse. + * - No runtime module under `src/` imports pdf-parse, and none imports + * pdfjs-dist (a type-only import included) except + * `src/lib/documents/pdfjs.ts`, so the text read and the rasterizer open + * documents with the same options. Comments are stripped first, so prose + * that names a package does not count. * * It does not prove the image carries one copy; the Dockerfile hoists the * pinned version by name and the tracing glob can only find what the lockfile - * installed. The behaviour itself is pinned by the two - * `pdfjs-one-runtime-*` tests under `src/lib/documents/__tests__/`. + * installed. The behaviour itself is pinned by the two `pdfjs-one-version-*` + * tests under `src/lib/documents/__tests__/`, one per order, each in a + * process of its own. */ import { createRequire } from "node:module"; import { join } from "node:path"; import { describe, expect, it } from "vitest"; -import { readSourceFolder, readSourceText } from "./helpers/source-files"; +import { + readSourceFolder, + readSourceText, + stripComments, +} from "./helpers/source-files"; const ROOT = process.cwd(); /** Relative to `src/`. */ @@ -84,27 +93,50 @@ function readLockfile(lockfile: string): { return { versions, rootImporters, unexplained }; } -/** Production modules under `src/` that import a pdf.js package. */ -function pdfjsImporters(files: Map): string[] { - const specifier = - /(?:\bfrom\s*|\bimport\s*\(\s*|\brequire\s*\(\s*|^\s*import\s*)["'](?:pdfjs-dist|pdf-parse)(?:\/[^"']*)?["']/m; +/** A static import, a re-export, a dynamic import or a require of `name`. */ +function importOf(name: string): RegExp { + return new RegExp( + String.raw`(?:\bfrom\s*|\bimport\s*\(\s*|\brequire\s*\(\s*|\bimport\s+)["']` + + name + + String.raw`(?:/[^"']*)?["']`, + ); +} +const PDFJS_IMPORT = importOf("pdfjs-dist"); +const PDF_PARSE_IMPORT = importOf("pdf-parse"); + +/** The modules whose comment-free source matches `pattern`, sorted. */ +function importers(files: Map, pattern: RegExp): string[] { return [...files] - .filter(([, source]) => specifier.test(source)) + .filter(([, source]) => pattern.test(source)) .map(([path]) => path) .sort(); } -/** Production modules under `src/`, keyed by their path relative to it. */ -function productionSources(): Map { +/** + * Runtime modules under `src/`, keyed by their path relative to it, with + * comments stripped. The suites, mocks and generated client are not loaded + * by the server. + */ +function runtimeSources(): Map { return new Map( readSourceFolder("src", { floor: 1000, - extensions: [".ts", ".tsx", ".mts", ".js", ".mjs"], + extensions: [ + ".ts", + ".tsx", + ".mts", + ".cts", + ".js", + ".jsx", + ".mjs", + ".cjs", + ], include: (rel) => !rel.startsWith("generated/") && - !rel.includes("__tests__/") && + !rel.split("/").includes("__tests__") && + !rel.split("/").includes("__mocks__") && !/\.(?:test|spec)\.[mc]?[tj]sx?$/.test(rel), - }), + }).map(([rel, source]) => [rel, stripComments(source)]), ); } @@ -132,10 +164,11 @@ describe("one pdf.js (#1277)", () => { expect(resolved.version).toBe(pinned); }); - it("only the shared loader imports pdf.js", () => { - const sources = productionSources(); + it("no runtime module imports pdf-parse, and only the shared loader imports pdf.js", () => { + const sources = runtimeSources(); expect(sources.has(LOADER), `${LOADER} exists`).toBe(true); - expect(pdfjsImporters(sources)).toEqual([LOADER]); + expect(importers(sources, PDF_PARSE_IMPORT)).toEqual([]); + expect(importers(sources, PDFJS_IMPORT)).toEqual([LOADER]); }); it("the readers catch what they exist to catch", () => { @@ -158,15 +191,32 @@ describe("one pdf.js (#1277)", () => { expect([...lock.versions].sort()).toEqual(["5.4.296", "6.4.299"]); expect(lock.unexplained).toEqual(["13: pdfjs-dist: 5.4.296"]); - const importers = pdfjsImporters( - new Map([ + for (const line of [ + 'import { PDFParse } from "pdf-parse";', + 'import type { PDFParse } from "pdf-parse";', + 'const { PDFParse } = await import("pdf-parse");', + "const m = require('pdf-parse/worker');", + 'export { PDFParse } from "pdf-parse";', + ]) { + expect(PDF_PARSE_IMPORT.test(line), line).toBe(true); + } + expect(PDF_PARSE_IMPORT.test('import x from "pdf-parser";')).toBe(false); + + const sources = new Map( + [ [LOADER, 'await import("pdfjs-dist/legacy/build/pdf.mjs")'], - ["a", 'import { PDFParse } from "pdf-parse";'], + [ + "a", + 'import type { TextItem } from "pdfjs-dist/types/src/display/api";', + ], ["b", "const m = await import(\n 'pdfjs-dist'\n);"], ["c", 'import "pdfjs-dist/legacy/build/pdf.worker.mjs";'], - ["d", '// mentions "pdfjs-dist" in prose only'], - ]), + ["d", '// once read with import("pdfjs-dist") here'], + ["e", 'const name = "pdfjs-dist";'], + ].map(([rel, source]) => [rel!, stripComments(source!)]), + ); + expect(importers(sources, PDFJS_IMPORT)).toEqual( + ["a", "b", "c", LOADER].sort(), ); - expect(importers).toEqual(["a", "b", "c", LOADER].sort()); }); }); diff --git a/src/lib/documents/__tests__/pdfjs-one-runtime-raster-first.test.ts b/src/lib/documents/__tests__/pdfjs-one-runtime-raster-first.test.ts deleted file mode 100644 index 1af8dc44f..000000000 --- a/src/lib/documents/__tests__/pdfjs-one-runtime-raster-first.test.ts +++ /dev/null @@ -1,26 +0,0 @@ -import { describe, it, vi } from "vitest"; - -/** - * GitHub #1277, rasterization first: a render used to claim the in-process - * pdf.js worker for its own copy, and every later local text read failed - * without a word, leaving the document out of the search index. See - * `pdfjs-one-runtime.ts`. - */ - -vi.mock("@/lib/logging/context", () => ({ annotate: vi.fn() })); -vi.mock("@/lib/logging/signal", () => ({ logCaught: vi.fn() })); - -import { - expectFreshProcess, - expectRaster, - expectTextRead, -} from "./pdfjs-one-runtime"; - -describe("one pdf.js per process (#1277): rasterization first", () => { - it("reads text after a rasterization, and rasterizes again after that", async () => { - expectFreshProcess(); - await expectRaster(); - await expectTextRead(); - await expectRaster(); - }); -}); diff --git a/src/lib/documents/__tests__/pdfjs-one-runtime-text-first.test.ts b/src/lib/documents/__tests__/pdfjs-one-runtime-text-first.test.ts deleted file mode 100644 index 846ec1852..000000000 --- a/src/lib/documents/__tests__/pdfjs-one-runtime-text-first.test.ts +++ /dev/null @@ -1,26 +0,0 @@ -import { describe, it, vi } from "vitest"; - -/** - * GitHub #1277, text first: a local text read used to claim the in-process - * pdf.js worker for its own copy, and every later rasterization (thumbnails, - * the vision read of a scan) failed until a restart. See - * `pdfjs-one-runtime.ts`. - */ - -vi.mock("@/lib/logging/context", () => ({ annotate: vi.fn() })); -vi.mock("@/lib/logging/signal", () => ({ logCaught: vi.fn() })); - -import { - expectFreshProcess, - expectRaster, - expectTextRead, -} from "./pdfjs-one-runtime"; - -describe("one pdf.js per process (#1277): text read first", () => { - it("rasterizes after a text read, and reads text again after that", async () => { - expectFreshProcess(); - await expectTextRead(); - await expectRaster(); - await expectTextRead(); - }); -}); diff --git a/src/lib/documents/__tests__/pdfjs-one-runtime.ts b/src/lib/documents/__tests__/pdfjs-one-runtime.ts deleted file mode 100644 index d3fb7a672..000000000 --- a/src/lib/documents/__tests__/pdfjs-one-runtime.ts +++ /dev/null @@ -1,50 +0,0 @@ -/** - * Shared steps for the two `pdfjs-one-runtime-*` files (GitHub #1277). - * - * Under Node, pdf.js runs its worker in-process and keeps it on - * `globalThis.pdfjsWorker`. Whichever pdf.js copy opens a document first - * owns that global for the life of the process, and a different copy then - * refuses every document with "The API version … does not match the Worker - * version …". The text reader and the rasterizer once loaded different - * copies, so whichever ran first broke the other until a restart. - * - * Each order needs a fresh process, because the global cannot be taken back - * once claimed; that is why the two orders live in two files (the isolated - * project gives each file its own process) instead of two tests here. - */ -import { readFileSync } from "node:fs"; -import { join } from "node:path"; - -import { expect } from "vitest"; - -import { extractPdfText } from "../local-extract"; -import { rasterizePdf } from "../rasterize-pdf"; - -const FIXTURE = join( - process.cwd(), - "tests", - "fixtures", - "pdf", - "fallback-fonts.pdf", -); - -const pdf = () => readFileSync(FIXTURE); - -/** Nothing in this process has opened a PDF yet. */ -export function expectFreshProcess(): void { - expect( - (globalThis as { pdfjsWorker?: unknown }).pdfjsWorker, - "the file must run in its own process; another file already opened a PDF", - ).toBeUndefined(); -} - -export async function expectTextRead(): Promise { - const result = await extractPdfText(pdf()); - expect(result).toMatchObject({ ok: true, source: "local-pdf" }); - if (result.ok) expect(result.text).toContain("Test Insurance AG"); -} - -export async function expectRaster(): Promise { - const result = await rasterizePdf(pdf(), 1); - expect(result).toMatchObject({ ok: true }); -} diff --git a/src/lib/documents/__tests__/pdfjs-one-version-extract-first.test.ts b/src/lib/documents/__tests__/pdfjs-one-version-extract-first.test.ts new file mode 100644 index 000000000..596523e00 --- /dev/null +++ b/src/lib/documents/__tests__/pdfjs-one-version-extract-first.test.ts @@ -0,0 +1,50 @@ +import { readFileSync } from "node:fs"; +import { join } from "node:path"; + +import { describe, expect, it, vi } from "vitest"; + +/** + * Text extraction and rasterising run in one process. In Node, pdfjs keeps its + * worker in one global, `globalThis.pdfjsWorker`. When `pdf-parse` brought its + * own pdfjs (5.4.296) beside the app's (6.4.299), the copy that ran first set + * that global, and the other copy refused it: "The API version "6.4.299" does + * not match the Worker version "5.4.296"". Thumbnails then failed as + * `raster-failed`, vision reads of a PDF answered 422, and the local text index + * stayed empty with no log line. The fix: read the text with the app's own + * pdfjs. + * + * This file extracts text first and rasterises second. Its twin in this + * folder runs the reverse order. Each order needs its own file, because a + * process keeps the first worker it loads. + * + * Mutation check: read the text through `pdf-parse` again in + * `extractPdfText` → the raster step here fails. + */ +vi.mock("@/lib/logging/context", () => ({ annotate: vi.fn() })); +vi.mock("@/lib/logging/signal", () => ({ logCaught: vi.fn() })); + +import { extractPdfText } from "../local-extract"; +import { rasterizePdf } from "../rasterize-pdf"; + +const fixture = readFileSync( + join(process.cwd(), "tests", "fixtures", "pdf", "fallback-fonts.pdf"), +); + +describe("one pdfjs for text and raster: text first", () => { + it("extracts, rasterises and extracts again in one process", async () => { + // The order only means something in a process that has not opened a PDF. + expect( + (globalThis as { pdfjsWorker?: unknown }).pdfjsWorker, + "this file must run in a process of its own", + ).toBeUndefined(); + + const first = await extractPdfText(fixture); + expect(first.ok).toBe(true); + + const raster = await rasterizePdf(fixture, 1); + expect(raster.ok).toBe(true); + + const again = await extractPdfText(fixture); + expect(again).toEqual(first); + }); +}); diff --git a/src/lib/documents/__tests__/pdfjs-one-version-raster-first.test.ts b/src/lib/documents/__tests__/pdfjs-one-version-raster-first.test.ts new file mode 100644 index 000000000..94b987be7 --- /dev/null +++ b/src/lib/documents/__tests__/pdfjs-one-version-raster-first.test.ts @@ -0,0 +1,50 @@ +import { readFileSync } from "node:fs"; +import { join } from "node:path"; + +import { describe, expect, it, vi } from "vitest"; + +/** + * Text extraction and rasterising run in one process. In Node, pdfjs keeps its + * worker in one global, `globalThis.pdfjsWorker`. When `pdf-parse` brought its + * own pdfjs (5.4.296) beside the app's (6.4.299), the copy that ran first set + * that global, and the other copy refused it: "The API version "6.4.299" does + * not match the Worker version "5.4.296"". Thumbnails then failed as + * `raster-failed`, vision reads of a PDF answered 422, and the local text index + * stayed empty with no log line. The fix: read the text with the app's own + * pdfjs. + * + * This file rasterises first and extracts text second. Its twin in this + * folder runs the reverse order. Each order needs its own file, because a + * process keeps the first worker it loads. + * + * Mutation check: read the text through `pdf-parse` again in + * `extractPdfText` → the text step here fails. + */ +vi.mock("@/lib/logging/context", () => ({ annotate: vi.fn() })); +vi.mock("@/lib/logging/signal", () => ({ logCaught: vi.fn() })); + +import { extractPdfText } from "../local-extract"; +import { rasterizePdf } from "../rasterize-pdf"; + +const fixture = readFileSync( + join(process.cwd(), "tests", "fixtures", "pdf", "fallback-fonts.pdf"), +); + +describe("one pdfjs for text and raster: raster first", () => { + it("rasterises, extracts and rasterises again in one process", async () => { + // The order only means something in a process that has not opened a PDF. + expect( + (globalThis as { pdfjsWorker?: unknown }).pdfjsWorker, + "this file must run in a process of its own", + ).toBeUndefined(); + + const first = await rasterizePdf(fixture, 1); + expect(first.ok).toBe(true); + + const text = await extractPdfText(fixture); + expect(text.ok).toBe(true); + + const again = await rasterizePdf(fixture, 1); + expect(again.ok).toBe(true); + }); +}); diff --git a/src/lib/documents/__tests__/pdfjs-text-layer.test.ts b/src/lib/documents/__tests__/pdfjs-text-layer.test.ts index aae80e893..ad331d608 100644 --- a/src/lib/documents/__tests__/pdfjs-text-layer.test.ts +++ b/src/lib/documents/__tests__/pdfjs-text-layer.test.ts @@ -1,80 +1,89 @@ import { jsPDF } from "jspdf"; +import type { + TextItem, + TextMarkedContent, +} from "pdfjs-dist/types/src/display/api"; import { describe, expect, it } from "vitest"; -import { pageTextLayer, readPdfText, type PdfTextItem } from "../pdfjs"; +import { joinTextRuns, readPdfText } from "../pdfjs"; /** - * The text layer as the content index reads it. The layout is the one the - * local text read produced before it moved onto the app's own pdf.js (#1277): - * a break where pdf.js marks one or the next item sits on a lower line, a tab - * between far-apart items on one line, each page followed by a blank line. + * The text layer as the content index reads it (#1277): runs in order, a line + * break where a run ends its line or the next one sits on another, a space + * where two runs on one line leave a gap, and nothing where they touch, so a + * word drawn in two fonts stays one word and a search for it still finds it. */ -/** A page whose viewport maps PDF space to itself, flipped like pdf.js. */ -function pageOf(items: PdfTextItem[]) { - return { - getViewport: () => ({ - width: 600, - height: 800, - convertToViewportPoint: (x: number, y: number) => [x, 800 - y], - }), - getTextContent: async () => ({ items }), - }; -} - -function item( +/** A 10 pt run starting at (x, y), 5 pt per character. */ +function run( str: string, x: number, y: number, - extra: Partial = {}, -): PdfTextItem { + extra: Partial = {}, +): TextItem { return { str, + dir: "ltr", transform: [10, 0, 0, 10, x, y], width: str.length * 5, height: 10, + fontName: "g_d0_f1", hasEOL: false, ...extra, }; } -describe("pageTextLayer", () => { - it("breaks once where pdf.js marks the end of a line", async () => { - const text = await pageTextLayer( - pageOf([ - item("Haemoglobin", 10, 700, { hasEOL: true }), - item("14.2 g/dL", 10, 688), +describe("joinTextRuns", () => { + it("keeps a word drawn in two fonts whole", () => { + expect( + joinTextRuns([ + run("Haem", 10, 700), + run("oglobin", 30, 700, { fontName: "g_d0_f2" }), ]), - ); - expect(text).toBe("Haemoglobin\n14.2 g/dL"); + ).toBe("Haemoglobin"); }); - it("breaks when the next item sits a line lower without a marked end", async () => { - const text = await pageTextLayer( - pageOf([item("Creatinine", 10, 700), item("0.9 mg/dL", 10, 680)]), + it("puts a space between runs on one line that leave a gap", () => { + expect(joinTextRuns([run("Glucose", 10, 700), run("92", 200, 700)])).toBe( + "Glucose 92", ); - expect(text).toBe("Creatinine\n0.9 mg/dL"); }); - it("does not break for a small baseline shift on one line", async () => { - const text = await pageTextLayer( - pageOf([item("10", 10, 700), item("3", 21, 703)]), - ); - expect(text).toBe("103"); + it("breaks once where a run ends its line", () => { + expect( + joinTextRuns([ + run("Haemoglobin", 10, 700, { hasEOL: true }), + run("14.2 g/dL", 10, 688), + ]), + ).toBe("Haemoglobin\n14.2 g/dL"); }); - it("separates far-apart items on one line with a tab", async () => { - const text = await pageTextLayer( - pageOf([item("Glucose", 10, 700), item("92", 200, 700)]), - ); - expect(text).toBe("Glucose\t92"); + it("breaks where the next run sits on another line without a marked end", () => { + expect( + joinTextRuns([run("Creatinine", 10, 700), run("0.9", 10, 680)]), + ).toBe("Creatinine\n0.9"); }); - it("skips marked-content items, which carry no text", async () => { - const text = await pageTextLayer( - pageOf([{ hasEOL: false } as PdfTextItem, item("Text", 10, 700)]), - ); - expect(text).toBe("Text"); + it("keeps a raised run on its line", () => { + expect(joinTextRuns([run("10", 10, 700), run("9", 20, 703)])).toBe("109"); + }); + + it("adds no blank line for an empty run that only ends a line", () => { + expect( + joinTextRuns([ + run("Line", 10, 700, { hasEOL: true }), + run("", 30, 700, { hasEOL: true }), + run("Next", 10, 688), + ]), + ).toBe("Line\nNext"); + }); + + it("skips marked-content markers, which carry no text", () => { + const marker: TextMarkedContent = { + type: "beginMarkedContent", + id: "mc0", + }; + expect(joinTextRuns([marker, run("Text", 10, 700)])).toBe("Text"); }); }); @@ -89,10 +98,10 @@ describe("readPdfText", () => { return Buffer.from(doc.output("arraybuffer")); } - it("joins pages with a blank line and adds no page markers", async () => { + it("reads the pages one after another and adds no page markers", async () => { const text = await readPdfText(pages(2)); expect(text).toBe( - "Page 1 lab report\nsecond line\n\nPage 2 lab report\nsecond line\n\n", + "Page 1 lab report\nsecond line\nPage 2 lab report\nsecond line", ); }); @@ -102,6 +111,17 @@ describe("readPdfText", () => { expect(text).not.toContain("Page 3"); }); + it("keeps a word whole that a real PDF draws in two fonts", async () => { + const doc = new jsPDF(); + doc.setFont("helvetica", "bold"); + doc.text("Haem", 10, 10); + const joint = 10 + doc.getTextWidth("Haem"); + doc.setFont("times", "normal"); + doc.text("oglobin 14.2 g/dL", joint, 10); + const text = await readPdfText(Buffer.from(doc.output("arraybuffer"))); + expect(text).toBe("Haemoglobin 14.2 g/dL"); + }); + it("throws what pdf.js throws on a file that is not a PDF", async () => { await expect(readPdfText(Buffer.from("not a pdf"))).rejects.toThrow(); }); diff --git a/src/lib/documents/content-index.ts b/src/lib/documents/content-index.ts index 8690260d7..d9f789186 100644 --- a/src/lib/documents/content-index.ts +++ b/src/lib/documents/content-index.ts @@ -325,7 +325,7 @@ export async function loadDocumentChatText( * - `vision` → an AI provider transcribed the stored original (image or PDF, * incl. scanned) — the AI-first primary path. * - `text-ocr` → browser-OCR text posted by the client (opt-in local OCR). - * - `local-pdf` → server-side text-layer extraction of a PDF (pdf.js), no + * - `local-pdf` → server-side text-layer extraction of a PDF (pdfjs), no * provider, no egress — the fallback when no provider is usable. * - `local-ocr` → server-side OCR of a scanned image/PDF (deferred follow-up). */ diff --git a/src/lib/documents/index-document.ts b/src/lib/documents/index-document.ts index 60afab046..aef4348de 100644 --- a/src/lib/documents/index-document.ts +++ b/src/lib/documents/index-document.ts @@ -24,7 +24,7 @@ * configured, the toggle is OFF for an external pick, the provider can't * read this file (e.g. a text-layer PDF on a non-Anthropic account), the * daily budget is reached, or the provider call fails — fall back to - * server-side text-layer extraction (pdf.js, no egress, milliseconds). + * server-side text-layer extraction (pdfjs, no egress, milliseconds). * So content search works at a baseline even with no AI, and every * text-layer PDF is searchable regardless of provider. * diff --git a/src/lib/documents/local-extract.ts b/src/lib/documents/local-extract.ts index b41598bc6..327f8a8b9 100644 --- a/src/lib/documents/local-extract.ts +++ b/src/lib/documents/local-extract.ts @@ -3,9 +3,18 @@ * * When NO AI provider is configured — or the user has not consented to one — * content search still needs a baseline. This module extracts a PDF's embedded - * text layer SERVER-SIDE through the app's one pdf.js (`./pdfjs`; reading the - * text layer never draws, so the native canvas does no work here), in - * milliseconds, with no OCR and no third-party egress. + * text layer SERVER-SIDE with pdfjs' `getTextContent` (the pure-JS/wasm core, + * never the native `@napi-rs/canvas`), in milliseconds, with no OCR and no + * third-party egress. + * + * It reads through `./pdfjs`, the one loader `./rasterize-pdf` opens documents + * with too, so both run the app's own `pdfjs-dist` with the same options. It + * used to go through `pdf-parse`, which brings a second pdfjs of its own, and + * the two cannot share a process: both take their worker from + * `globalThis.pdfjsWorker`, so whichever ran first broke the other ("The API + * version "6.4.299" does not match the Worker version "5.4.296""). Thumbnails + * and vision reads failed after a text extraction, and the text extraction + * failed after a raster. * The extracted text feeds the SAME blind, encrypted content index the provider * path writes (`upsertContentIndex`), so nothing readable is stored at rest. * @@ -62,11 +71,10 @@ export type LocalExtractResult = export async function extractPdfText( buffer: Buffer, ): Promise { - let raw: string; + let text: string; try { - // Bounded page walk; pages joined by a blank line, with no page markers, - // so only real document text is tokenised. - raw = await readPdfText(buffer, LOCAL_MAX_PAGES); + // Bound the page walk so a pathological many-page PDF costs no more. + text = (await readPdfText(buffer, LOCAL_MAX_PAGES)).trim(); } catch (err) { const errorType = err instanceof Error ? err.name || "Error" : typeof err; annotate({ @@ -78,7 +86,6 @@ export async function extractPdfText( }); return { ok: false, reason: "error" }; } - const text = raw.trim(); if (text.length < LOCAL_TEXT_MIN_CHARS) { return { ok: false, reason: "empty" }; } diff --git a/src/lib/documents/pdfjs.ts b/src/lib/documents/pdfjs.ts index 5d44c1f31..9409323e9 100644 --- a/src/lib/documents/pdfjs.ts +++ b/src/lib/documents/pdfjs.ts @@ -1,5 +1,6 @@ /** - * The one pdf.js this server runs, and the one way it is opened. + * The one pdf.js this server runs, the one way it is opened, and the one way + * its text layer is read. * * Under Node, pdf.js does not start a worker thread: it loads its worker code * into the same process and keeps it on `globalThis.pdfjsWorker`. Whichever @@ -15,7 +16,9 @@ * reads the browser-only `DOMMatrix` global, so a static import would make the * server chunk evaluate it at instantiation and throw `DOMMatrix is not * defined`, taking down every route and worker sharing the chunk. The import - * resolves once and is module-cached after that. + * resolves once and is module-cached after that. The type imports below are + * erased at build, so they load nothing; they let the compiler check every + * call here against pdf.js' own API. * * This module has no logging of its own, so the end-to-end suite can read * report text with it; the callers say what failed. @@ -25,74 +28,11 @@ import { existsSync } from "node:fs"; import { createRequire } from "node:module"; import { dirname, join } from "node:path"; -// Structural types for the slice of the pdf.js API used here and in the -// rasterizer. Type-only, so nothing evaluates pdf.js for its types. -export interface PdfViewport { - width: number; - height: number; - convertToViewportPoint(x: number, y: number): number[]; -} -export interface PdfCanvas { - width: number; - height: number; - toBuffer(mime: "image/jpeg", quality?: number): Buffer; -} -export interface PdfContext2D { - getImageData( - x: number, - y: number, - width: number, - height: number, - ): { data: Uint8ClampedArray }; -} -interface PdfCanvasAndContext { - canvas: PdfCanvas; - context: PdfContext2D; -} -interface PdfCanvasFactory { - create(width: number, height: number): PdfCanvasAndContext; - destroy?(target: PdfCanvasAndContext): void; -} -/** A text-layer item; marked-content items carry no `str` and are skipped. */ -export interface PdfTextItem { - str?: string; - hasEOL?: boolean; - transform?: number[]; - width?: number; - height?: number; -} -export interface PdfPage { - getViewport(opts: { scale: number }): PdfViewport; - render(opts: { - canvasContext: unknown; - viewport: PdfViewport; - canvas: unknown; - }): { promise: Promise }; - cleanup(): void; - getTextContent(): Promise<{ items: PdfTextItem[] }>; -} -export interface PdfDocument { - numPages: number; - canvasFactory: PdfCanvasFactory; - getPage(pageNumber: number): Promise; -} -export interface PdfLoadingTask { - promise: Promise; - destroy(): Promise; -} -interface PdfDocumentOptions { - data: Uint8Array; - verbosity?: number; - isEvalSupported?: boolean; - useSystemFonts?: boolean; - standardFontDataUrl?: string; - cMapUrl?: string; - cMapPacked?: boolean; - wasmUrl?: string; -} -interface PdfjsModule { - getDocument(opts: PdfDocumentOptions): PdfLoadingTask; -} +import type { PDFDocumentLoadingTask } from "pdfjs-dist"; +import type { + TextItem, + TextMarkedContent, +} from "pdfjs-dist/types/src/display/api"; /** * Where pdf.js finds the data it does not carry in its JS: the standard 14 @@ -169,15 +109,13 @@ export function __resetPdfjsAssetsForTests(): void { */ export async function openPdfDocument( data: Uint8Array | Buffer, -): Promise { - const pdfjs = - (await import("pdfjs-dist/legacy/build/pdf.mjs")) as unknown as PdfjsModule; +): Promise { + const pdfjs = await import("pdfjs-dist/legacy/build/pdf.mjs"); const assets = resolvePdfjsAssets(); return pdfjs.getDocument({ // A copy: pdf.js transfers the buffer it is given to its worker. data: new Uint8Array(data), verbosity: 0, // VerbosityLevel.ERRORS: no console noise on odd PDFs. - isEvalSupported: false, // never eval font programs (defence in depth). // There are no system fonts to use: substitutes come from pdf.js' own // standard font data (and, when rendering, the registered fallbacks). useSystemFonts: false, @@ -192,73 +130,83 @@ export async function openPdfDocument( }); } -/** Vertical gap (pt) past which the next item starts a new line. */ -const LINE_THRESHOLD = 4.6; -/** Horizontal gap (pt) on one line past which the next item is a new cell. */ -const CELL_THRESHOLD = 7; - /** - * One page's text layer as plain text: items in content order, a line break - * where pdf.js marks one or where the next item sits on a lower line, and a - * tab between items on one line that are far apart (table cells). - * - * This is the layout `pdf-parse` produced with its defaults, kept so that - * swapping it out did not change what the content index sees. + * Where a text run sits on the page. pdfjs gives each run a text matrix + * `[a, b, c, d, x, y]`: `x` and `y` are where the run starts, and `d` is its + * vertical scale, which for upright text is the font size. */ -export async function pageTextLayer( - page: Pick, -): Promise { - const viewport = page.getViewport({ scale: 1 }); - const { items } = await page.getTextContent(); - const parts: string[] = []; - let lastX: number | undefined; - let lastY: number | undefined; - let lineHeight = 0; - for (const item of items) { - if (typeof item.str !== "string") continue; - const transform = item.transform ?? [1, 0, 0, 1, 0, 0]; - const [x = 0, y = 0] = viewport.convertToViewportPoint( - transform[4] ?? 0, - transform[5] ?? 0, - ); - let str = item.str; - if (lastY !== undefined) { - const gap = Math.abs(lastY - y); - if (gap > LINE_THRESHOLD) { - const previous = parts.at(-1); - const opensLine = - str.startsWith("\n") || (str.trim() === "" && item.hasEOL === true); - if ( - previous !== undefined && - !previous.endsWith("\n") && - !opensLine && - gap - 1 > lineHeight - ) { - parts.push("\n"); - lineHeight = 0; - } - } else if ( - gap < LINE_THRESHOLD && - lastX !== undefined && - Math.abs(lastX - x) > CELL_THRESHOLD - ) { - str = `\t${str}`; - } - } - parts.push(str); - lastX = x + (item.width ?? 0); - lastY = y; - lineHeight = Math.max(lineHeight, item.height ?? 0); - if (item.hasEOL) parts.push("\n"); - if (item.hasEOL || str.endsWith("\n")) lineHeight = 0; - } - return parts.join(""); +interface RunPosition { + x: number; + /** The baseline the run sits on. */ + y: number; + /** Where the run ends: its start plus its width. */ + end: number; + fontSize: number; +} + +function positionOf(run: TextItem): RunPosition { + const [, , , scaleY, x = 0, y = 0] = run.transform as number[]; + return { + x, + y, + end: x + run.width, + fontSize: Math.abs(scaleY ?? 0) || run.height || 1, + }; } /** - * The text layer of the first `maxPages` pages, each page followed by a blank - * line. Throws what pdf.js throws (a malformed file, a password, a page that - * will not parse); the document is torn down either way. + * The next run is on a new line when its baseline moved by more than half a + * font size. Runs on one line can sit a little higher or lower (a superscript, + * a mixed font), so a small shift does not count. + */ +function startsNewLine(previous: RunPosition, next: RunPosition): boolean { + return Math.abs(next.y - previous.y) > previous.fontSize / 2; +} + +/** + * Two runs on one line are separate words when the gap between them is wider + * than a tenth of the font size. Runs that touch, such as a word drawn in two + * fonts, stay one word. + */ +function leavesWordGap(previous: RunPosition, next: RunPosition): boolean { + return next.x - previous.end > next.fontSize / 10; +} + +/** + * One page's text: its runs in order, with a line break where a run ends a + * line or the next one starts a new line, and a space where two runs on one + * line leave a gap between words. + */ +export function joinTextRuns( + items: Array, +): string { + let text = ""; + let previous: RunPosition | null = null; + + for (const item of items) { + // Marked-content markers carry no text. + if (!("str" in item)) continue; + const position = positionOf(item); + + if (previous && !text.endsWith("\n")) { + if (startsNewLine(previous, position)) { + text += "\n"; + } else if (leavesWordGap(previous, position) && !/\s$/.test(text)) { + text += " "; + } + } + text += item.str; + if (item.hasEOL && !text.endsWith("\n")) text += "\n"; + + previous = position; + } + return text; +} + +/** + * The text layer of the first `maxPages` pages, one page after another. + * Throws what pdf.js throws (a malformed file, a password, a page that will + * not parse); the document is torn down either way. */ export async function readPdfText( data: Uint8Array | Buffer, @@ -267,17 +215,17 @@ export async function readPdfText( const task = await openPdfDocument(data); try { const doc = await task.promise; - const last = Math.min(doc.numPages, maxPages); - let text = ""; - for (let pageNumber = 1; pageNumber <= last; pageNumber++) { - const page = await doc.getPage(pageNumber); + const pageCount = Math.min(doc.numPages, maxPages); + const pages: string[] = []; + for (let n = 1; n <= pageCount; n++) { + const page = await doc.getPage(n); try { - text += `${await pageTextLayer(page)}\n\n`; + pages.push(joinTextRuns((await page.getTextContent()).items)); } finally { page.cleanup(); } } - return text; + return pages.join("\n"); } finally { try { await task.destroy(); diff --git a/src/lib/documents/rasterize-pdf.ts b/src/lib/documents/rasterize-pdf.ts index f23730e14..5bb98f32a 100644 --- a/src/lib/documents/rasterize-pdf.ts +++ b/src/lib/documents/rasterize-pdf.ts @@ -33,15 +33,7 @@ import { Buffer } from "node:buffer"; import { annotate } from "@/lib/logging/context"; import { nativeCanvasSupported } from "./native-canvas-support"; -import { - openPdfDocument, - resolvePdfjsAssets, - type PdfContext2D, - type PdfDocument, - type PdfjsAssets, - type PdfLoadingTask, - type PdfPage, -} from "./pdfjs"; +import { openPdfDocument, resolvePdfjsAssets, type PdfjsAssets } from "./pdfjs"; /** * Whether the PDF rasterizer is available in this build. The native @@ -114,6 +106,33 @@ export type RasterResult = } | { ok: false; reason: "unsupported" | "render-failed" }; +/** + * The canvas pdf.js draws into under Node is `@napi-rs/canvas`. pdf.js' own + * declarations are written for the DOM and type the document's canvas factory + * as a bare `object`, so the slice used here is named: a DOM-shaped canvas + * and 2D context, plus the JPEG encoder the native canvas adds. + */ +interface RenderTarget { + canvas: HTMLCanvasElement & { + toBuffer(mime: "image/jpeg", quality?: number): Buffer; + }; + context: CanvasRenderingContext2D; +} +interface RenderTargetFactory { + create(width: number, height: number): RenderTarget; + destroy?(target: RenderTarget): void; +} + +/** + * A page's text layer, as far as `pageText` reads it: text runs, and + * marked-content markers that carry only a `type` and no text. + */ +interface TextLayerPage { + getTextContent(): Promise<{ + items: Array<{ str?: string; hasEOL?: boolean; type?: string }>; + }>; +} + interface GlobalFontsLike { registerFromPath(path: string, nameAlias?: string): unknown; } @@ -144,7 +163,11 @@ async function registerFallbackFamilies(assets: PdfjsAssets): Promise { } /** Share of dark pixels on a rendered page (luminance below half). */ -function inkRatio(context: PdfContext2D, width: number, height: number) { +function inkRatio( + context: Pick, + width: number, + height: number, +) { const { data } = context.getImageData(0, 0, width, height); let dark = 0; for (let i = 0; i < data.length; i += 4) { @@ -188,9 +211,7 @@ function headThatFits(text: string, room: number): string { * `RASTER_PAGE_TEXT_MAX` it is cut (at a line, else a space, else hard) and * marked. */ -export async function pageText( - page: Pick, -): Promise { +export async function pageText(page: TextLayerPage): Promise { const max = RASTER_PAGE_TEXT_MAX; try { const { items } = await page.getTextContent(); @@ -248,8 +269,7 @@ export async function rasterizePdf( maxPages: number = RASTER_MAX_PAGES, opts: { withText?: boolean } = {}, ): Promise { - let doc: PdfDocument | null = null; - let task: PdfLoadingTask | null = null; + let task: Awaited> | null = null; // CPU-feature gate: pdfjs renders through @napi-rs/canvas (Skia), whose x64 // build uses AVX2 — an unsupported CPU dies with an uncatchable SIGILL on // the first render. Degrade to the existing no-raster path instead (scanned @@ -273,7 +293,7 @@ export async function rasterizePdf( } task = await openPdfDocument(buffer); - doc = await task.promise; + const doc = await task.promise; // Never exceed the standing DoS ceiling, even if a caller asks for more. const cap = Math.min(Math.max(1, maxPages), RASTER_HARD_MAX_PAGES); @@ -292,7 +312,7 @@ export async function rasterizePdf( longEdge > 0 ? Math.min(1, RASTER_TARGET_LONG_EDGE / longEdge) : 1; const viewport = page.getViewport({ scale }); - const factory = doc.canvasFactory; + const factory = doc.canvasFactory as RenderTargetFactory; const cc = factory.create( Math.ceil(viewport.width), Math.ceil(viewport.height),