Merge pull request #1278

Read a PDF's text with the app's own pdfjs (#1277).

src/lib/documents/__tests__/pdfjs-one-version-extract-first.test.ts

@@ -0,0 +1,43 @@
+import { readFileSync } from "node:fs";
+import { join } from "node:path";
+
+import { describe, expect, it, vi } from "vitest";
+
+/**
+ * Text extraction and rasterising run in one process. In Node, pdfjs keeps its
+ * worker in one global, `globalThis.pdfjsWorker`. When `pdf-parse` brought its
+ * own pdfjs (5.4.296) beside the app's (6.4.299), the copy that ran first set
+ * that global, and the other copy refused it: "The API version "6.4.299" does
+ * not match the Worker version "5.4.296"". Thumbnails then failed as
+ * `raster-failed`, vision reads of a PDF answered 422, and the local text index
+ * stayed empty with no log line. The fix: read the text with the app's own
+ * pdfjs.
+ *
+ * This file extracts text first and rasterises second. Its twin in this
+ * folder runs the reverse order. Each order needs its own file, because a
+ * process keeps the first worker it loads.
+ *
+ * Mutation check: read the text through `pdf-parse` again in
+ * `extractPdfText` → the raster step here fails.
+ */
+vi.mock("@/lib/logging/context", () => ({ annotate: vi.fn() }));
+
+import { extractPdfText } from "../local-extract";
+import { rasterizePdf } from "../rasterize-pdf";
+
+const fixture = readFileSync(
+  join(process.cwd(), "tests", "fixtures", "pdf", "fallback-fonts.pdf"),
+);
+
+describe("one pdfjs for text and raster: text first", () => {
+  it("extracts, rasterises and extracts again in one process", async () => {
+    const first = await extractPdfText(fixture);
+    expect(first.ok).toBe(true);
+
+    const raster = await rasterizePdf(fixture, 1);
+    expect(raster.ok).toBe(true);
+
+    const again = await extractPdfText(fixture);
+    expect(again).toEqual(first);
+  });
+});

src/lib/documents/__tests__/pdfjs-one-version-raster-first.test.ts

@@ -0,0 +1,43 @@
+import { readFileSync } from "node:fs";
+import { join } from "node:path";
+
+import { describe, expect, it, vi } from "vitest";
+
+/**
+ * Text extraction and rasterising run in one process. In Node, pdfjs keeps its
+ * worker in one global, `globalThis.pdfjsWorker`. When `pdf-parse` brought its
+ * own pdfjs (5.4.296) beside the app's (6.4.299), the copy that ran first set
+ * that global, and the other copy refused it: "The API version "6.4.299" does
+ * not match the Worker version "5.4.296"". Thumbnails then failed as
+ * `raster-failed`, vision reads of a PDF answered 422, and the local text index
+ * stayed empty with no log line. The fix: read the text with the app's own
+ * pdfjs.
+ *
+ * This file rasterises first and extracts text second. Its twin in this
+ * folder runs the reverse order. Each order needs its own file, because a
+ * process keeps the first worker it loads.
+ *
+ * Mutation check: read the text through `pdf-parse` again in
+ * `extractPdfText` → the text step here fails.
+ */
+vi.mock("@/lib/logging/context", () => ({ annotate: vi.fn() }));
+
+import { extractPdfText } from "../local-extract";
+import { rasterizePdf } from "../rasterize-pdf";
+
+const fixture = readFileSync(
+  join(process.cwd(), "tests", "fixtures", "pdf", "fallback-fonts.pdf"),
+);
+
+describe("one pdfjs for text and raster: raster first", () => {
+  it("rasterises, extracts and rasterises again in one process", async () => {
+    const first = await rasterizePdf(fixture, 1);
+    expect(first.ok).toBe(true);
+
+    const text = await extractPdfText(fixture);
+    expect(text.ok).toBe(true);
+
+    const again = await rasterizePdf(fixture, 1);
+    expect(again.ok).toBe(true);
+  });
+});

src/lib/documents/content-index.ts

@@ -325,7 +325,7 @@ export async function loadDocumentChatText(
  *   - `vision`    → an AI provider transcribed the stored original (image or PDF,
  *                   incl. scanned) — the AI-first primary path.
  *   - `text-ocr`  → browser-OCR text posted by the client (opt-in local OCR).
- *   - `local-pdf` → server-side text-layer extraction of a PDF (pdf.js), no
+ *   - `local-pdf` → server-side text-layer extraction of a PDF (pdfjs), no
  *                   provider, no egress — the fallback when no provider is usable.
  *   - `local-ocr` → server-side OCR of a scanned image/PDF (deferred follow-up).
  */

src/lib/documents/index-document.ts

@@ -24,7 +24,7 @@
  *      configured, the toggle is OFF for an external pick, the provider can't
  *      read this file (e.g. a text-layer PDF on a non-Anthropic account), the
  *      daily budget is reached, or the provider call fails — fall back to
- *      server-side text-layer extraction (pdf.js, no egress, milliseconds).
+ *      server-side text-layer extraction (pdfjs, no egress, milliseconds).
  *      So content search works at a baseline even with no AI, and every
  *      text-layer PDF is searchable regardless of provider.
  *

src/lib/documents/local-extract.ts

@@ -3,9 +3,17 @@
  *
  * When NO AI provider is configured — or the user has not consented to one —
  * content search still needs a baseline. This module extracts a PDF's embedded
- * text layer SERVER-SIDE through the app's one pdf.js (`./pdfjs`; reading the
- * text layer never draws, so the native canvas does no work here), in
- * milliseconds, with no OCR and no third-party egress.
+ * text layer SERVER-SIDE with pdfjs' `getTextContent` (the pure-JS/wasm core,
+ * never the native `@napi-rs/canvas`), in milliseconds, with no OCR and no
+ * third-party egress.
+ *
+ * It reads through the app's own `pdfjs-dist`, the copy `./rasterize-pdf`
+ * renders with. It used to go through `pdf-parse`, which brings a second
+ * pdfjs of its own, and the two cannot share a process: both take their
+ * worker from `globalThis.pdfjsWorker`, so whichever ran first broke the
+ * other ("The API version "6.4.299" does not match the Worker version
+ * "5.4.296""). Thumbnails and vision reads failed after a text extraction,
+ * and the text extraction failed after a raster.
  * The extracted text feeds the SAME blind, encrypted content index the provider
  * path writes (`upsertContentIndex`), so nothing readable is stored at rest.
  *
@@ -20,10 +28,92 @@
  */
 import { Buffer } from "node:buffer";
 
-import { annotate } from "@/lib/logging/context";
-import { logCaught } from "@/lib/logging/signal";
+import type { PDFDocumentLoadingTask } from "pdfjs-dist";
+import type {
+  TextItem,
+  TextMarkedContent,
+} from "pdfjs-dist/types/src/display/api";
 
-import { readPdfText } from "./pdfjs";
+import { annotate } from "@/lib/logging/context";
+
+// pdfjs' module top-level references the browser-only `DOMMatrix` global. A
+// STATIC import makes the Turbopack server chunk evaluate that reference at
+// instantiation and throw `DOMMatrix is not defined`, taking down every
+// route/worker that shares the chunk (the document routes among them). The
+// module is therefore loaded lazily inside `extractPdfText` via a runtime
+// `import()`. The type imports above are erased at build, so they load
+// nothing; they let the compiler check this file against pdfjs' own API.
+
+/**
+ * Where a text run sits on the page. pdfjs gives each run a text matrix
+ * `[a, b, c, d, x, y]`: `x` and `y` are where the run starts, and `d` is its
+ * vertical scale, which for upright text is the font size.
+ */
+interface RunPosition {
+  x: number;
+  /** The baseline the run sits on. */
+  y: number;
+  /** Where the run ends: its start plus its width. */
+  end: number;
+  fontSize: number;
+}
+
+function positionOf(run: TextItem): RunPosition {
+  const [, , , scaleY, x = 0, y = 0] = run.transform as number[];
+  return {
+    x,
+    y,
+    end: x + run.width,
+    fontSize: Math.abs(scaleY ?? 0) || run.height || 1,
+  };
+}
+
+/**
+ * The next run is on a new line when its baseline moved by more than half a
+ * font size. Runs on one line can sit a little higher or lower (a superscript,
+ * a mixed font), so a small shift does not count.
+ */
+function startsNewLine(previous: RunPosition, next: RunPosition): boolean {
+  return Math.abs(next.y - previous.y) > previous.fontSize / 2;
+}
+
+/**
+ * Two runs on one line are separate words when the gap between them is wider
+ * than a tenth of the font size. Runs that touch, such as a word drawn in two
+ * fonts, stay one word.
+ */
+function leavesWordGap(previous: RunPosition, next: RunPosition): boolean {
+  return next.x - previous.end > next.fontSize / 10;
+}
+
+/**
+ * One page's text: its runs in order, with a line break where a run ends a
+ * line or the next one starts a new line, and a space where two runs on one
+ * line leave a gap between words.
+ */
+function pageText(items: Array<TextItem | TextMarkedContent>): string {
+  let text = "";
+  let previous: RunPosition | null = null;
+
+  for (const item of items) {
+    // Marked-content markers carry no text.
+    if (!("str" in item)) continue;
+    const position = positionOf(item);
+
+    if (previous && !text.endsWith("\n")) {
+      if (startsNewLine(previous, position)) {
+        text += "\n";
+      } else if (leavesWordGap(previous, position) && !/\s$/.test(text)) {
+        text += " ";
+      }
+    }
+    text += item.str;
+    if (item.hasEOL && !text.endsWith("\n")) text += "\n";
+
+    previous = position;
+  }
+  return text;
+}
 
 /**
  * Minimum joined text length for a PDF to count as a real text-layer document.
@@ -55,34 +145,49 @@ export type LocalExtractResult =
  * Extract a text-layer PDF's embedded text server-side. Returns `empty` for a
  * scanned/no-text-layer PDF and `error` for a malformed/encrypted one — never
  * throws, so a bad document can never abort the caller (the upload/backfill job
- * just leaves it un-indexed). An `error` leaves a `warn` line naming what
- * pdf.js threw, because the outcome alone does not say why a document stayed
- * out of the index.
+ * just leaves it un-indexed).
  */
 export async function extractPdfText(
   buffer: Buffer,
 ): Promise<LocalExtractResult> {
-  let raw: string;
+  let task: PDFDocumentLoadingTask | null = null;
   try {
-    // Bounded page walk; pages joined by a blank line, with no page markers,
-    // so only real document text is tokenised.
-    raw = await readPdfText(buffer, LOCAL_MAX_PAGES);
-  } catch (err) {
-    const errorType = err instanceof Error ? err.name || "Error" : typeof err;
-    annotate({
-      action: { name: "documents.localIndex.extractFailed" },
-      meta: { localExtractError: errorType },
-    });
-    logCaught("documents.local_extract.failed", err, {
-      bytes: buffer.length,
+    // Lazy load: keeps pdfjs/DOMMatrix out of the eager server chunk (see the
+    // import-site note). Resolved at first PDF extraction, then module-cached.
+    const pdfjs = await import("pdfjs-dist/legacy/build/pdf.mjs");
+    task = pdfjs.getDocument({
+      data: new Uint8Array(buffer),
+      verbosity: 0, // VerbosityLevel.ERRORS — no console spam on odd PDFs.
+      useSystemFonts: false,
     });
+    const doc = await task.promise;
+    // Bound the page walk so a pathological many-page PDF costs no more.
+    const pageCount = Math.min(doc.numPages, LOCAL_MAX_PAGES);
+    const pages: string[] = [];
+    for (let n = 1; n <= pageCount; n++) {
+      const page = await doc.getPage(n);
+      try {
+        pages.push(pageText((await page.getTextContent()).items));
+      } finally {
+        page.cleanup();
+      }
+    }
+    const text = pages.join("\n").trim();
+    if (text.length < LOCAL_TEXT_MIN_CHARS) {
+      return { ok: false, reason: "empty" };
+    }
+    return { ok: true, text, source: "local-pdf" };
+  } catch {
     return { ok: false, reason: "error" };
+  } finally {
+    if (task) {
+      try {
+        await task.destroy();
+      } catch {
+        // Best-effort teardown; a destroy failure must not mask the result.
+      }
+    }
   }
-  const text = raw.trim();
-  if (text.length < LOCAL_TEXT_MIN_CHARS) {
-    return { ok: false, reason: "empty" };
-  }
-  return { ok: true, text, source: "local-pdf" };
 }
 
 /**