fix(documents): read a PDF's text with the app's own pdfjs

Local text extraction went through pdf-parse, which pins its own
pdfjs-dist (5.4.296) beside the app's (6.4.299), and both copies take
their worker from globalThis.pdfjsWorker. Whichever ran first in a process
broke the other: after a text extraction every raster failed with "The
API version "6.4.299" does not match the Worker version "5.4.296""
(thumbnails raster-failed, vision reads of a PDF 422 extractFailed), and
after a raster the text extraction failed and the local content index
stayed empty without a trace.

extractPdfText now reads the text layer with getTextContent on the
pdfjs-dist the rasterizer uses, so only one pdfjs runs in the server.
Pointing pdf-parse at 6.4 instead is not enough: its destroy() calls
PDFDocumentProxy.destroy(), which pdfjs 6 no longer has, so every
document would skip its teardown. Two tests run text and raster in one
process, each order in its own file.

src/lib/documents/__tests__/pdfjs-one-version-extract-first.test.ts

@@ -0,0 +1,40 @@
+import { readFileSync } from "node:fs";
+import { join } from "node:path";
+
+import { describe, expect, it, vi } from "vitest";
+
+/**
+ * Text extraction and rasterising share one process, and pdfjs keeps its worker
+ * on `globalThis.pdfjsWorker`. While `pdf-parse` brought its own pdfjs
+ * (5.4.296) beside the app's (6.4.299), whichever ran first installed its
+ * worker and the other refused it: "The API version "6.4.299" does not match
+ * the Worker version "5.4.296"". Thumbnails failed as `raster-failed`, vision
+ * reads of a PDF answered 422 and the local text index stayed empty without a
+ * word. Reading the text through the app's own pdfjs is the fix; this file runs
+ * extraction first and `pdfjs-one-version-raster-first.test.ts` the reverse,
+ * each in a fresh process state.
+ *
+ * Mutation check: read the text through `pdf-parse` again in `extractPdfText` →
+ * the raster step here fails.
+ */
+vi.mock("@/lib/logging/context", () => ({ annotate: vi.fn() }));
+
+import { extractPdfText } from "../local-extract";
+import { rasterizePdf } from "../rasterize-pdf";
+
+const fixture = readFileSync(
+  join(process.cwd(), "tests", "fixtures", "pdf", "fallback-fonts.pdf"),
+);
+
+describe("one pdfjs for text and raster: text first", () => {
+  it("extracts, rasterises and extracts again in one process", async () => {
+    const first = await extractPdfText(fixture);
+    expect(first.ok).toBe(true);
+
+    const raster = await rasterizePdf(fixture, 1);
+    expect(raster.ok).toBe(true);
+
+    const again = await extractPdfText(fixture);
+    expect(again).toEqual(first);
+  });
+});

src/lib/documents/__tests__/pdfjs-one-version-raster-first.test.ts

@@ -0,0 +1,40 @@
+import { readFileSync } from "node:fs";
+import { join } from "node:path";
+
+import { describe, expect, it, vi } from "vitest";
+
+/**
+ * Text extraction and rasterising share one process, and pdfjs keeps its worker
+ * on `globalThis.pdfjsWorker`. While `pdf-parse` brought its own pdfjs
+ * (5.4.296) beside the app's (6.4.299), whichever ran first installed its
+ * worker and the other refused it: "The API version "6.4.299" does not match
+ * the Worker version "5.4.296"". Thumbnails failed as `raster-failed`, vision
+ * reads of a PDF answered 422 and the local text index stayed empty without a
+ * word. Reading the text through the app's own pdfjs is the fix; the twin runs
+ * extraction first (`pdfjs-one-version-extract-first.test.ts`) and this file
+ * the reverse, each in a fresh process state.
+ *
+ * Mutation check: read the text through `pdf-parse` again in `extractPdfText` →
+ * the text step here fails.
+ */
+vi.mock("@/lib/logging/context", () => ({ annotate: vi.fn() }));
+
+import { extractPdfText } from "../local-extract";
+import { rasterizePdf } from "../rasterize-pdf";
+
+const fixture = readFileSync(
+  join(process.cwd(), "tests", "fixtures", "pdf", "fallback-fonts.pdf"),
+);
+
+describe("one pdfjs for text and raster: raster first", () => {
+  it("rasterises, extracts and rasterises again in one process", async () => {
+    const first = await rasterizePdf(fixture, 1);
+    expect(first.ok).toBe(true);
+
+    const text = await extractPdfText(fixture);
+    expect(text.ok).toBe(true);
+
+    const again = await rasterizePdf(fixture, 1);
+    expect(again.ok).toBe(true);
+  });
+});

src/lib/documents/content-index.ts

@@ -325,7 +325,7 @@ export async function loadDocumentChatText(
  *   - `vision`    → an AI provider transcribed the stored original (image or PDF,
  *                   incl. scanned) — the AI-first primary path.
  *   - `text-ocr`  → browser-OCR text posted by the client (opt-in local OCR).
- *   - `local-pdf` → server-side text-layer extraction of a PDF (`pdf-parse`), no
+ *   - `local-pdf` → server-side text-layer extraction of a PDF (pdfjs), no
  *                   provider, no egress — the fallback when no provider is usable.
  *   - `local-ocr` → server-side OCR of a scanned image/PDF (deferred follow-up).
  */

src/lib/documents/index-document.ts

@@ -24,7 +24,7 @@
  *      configured, the toggle is OFF for an external pick, the provider can't
  *      read this file (e.g. a text-layer PDF on a non-Anthropic account), the
  *      daily budget is reached, or the provider call fails — fall back to
- *      server-side text-layer extraction (`pdf-parse`, no egress, milliseconds).
+ *      server-side text-layer extraction (pdfjs, no egress, milliseconds).
  *      So content search works at a baseline even with no AI, and every
  *      text-layer PDF is searchable regardless of provider.
  *

src/lib/documents/local-extract.ts

@@ -3,9 +3,17 @@
  *
  * When NO AI provider is configured — or the user has not consented to one —
  * content search still needs a baseline. This module extracts a PDF's embedded
- * text layer SERVER-SIDE with `pdf-parse` (already a direct dependency; its
- * `getText()` touches only the pure-JS/wasm pdfjs core, never the native
- * `@napi-rs/canvas`), in milliseconds, with no OCR and no third-party egress.
+ * text layer SERVER-SIDE with pdfjs' `getTextContent` (the pure-JS/wasm core,
+ * never the native `@napi-rs/canvas`), in milliseconds, with no OCR and no
+ * third-party egress.
+ *
+ * It reads through the app's own `pdfjs-dist`, the copy `./rasterize-pdf`
+ * renders with. It used to go through `pdf-parse`, which brings a second
+ * pdfjs of its own, and the two cannot share a process: both take their
+ * worker from `globalThis.pdfjsWorker`, so whichever ran first broke the
+ * other ("The API version "6.4.299" does not match the Worker version
+ * "5.4.296""). Thumbnails and vision reads failed after a text extraction,
+ * and the text extraction failed after a raster.
  * The extracted text feeds the SAME blind, encrypted content index the provider
  * path writes (`upsertContentIndex`), so nothing readable is stored at rest.
  *
@@ -20,19 +28,68 @@
  */
 import { Buffer } from "node:buffer";
 
-// `pdf-parse` pulls in pdfjs, whose module top-level references the browser-only
-// `DOMMatrix` global. A STATIC import makes the Turbopack server chunk evaluate
-// that reference at instantiation and throw `DOMMatrix is not defined`, taking
-// down every route/worker that shares the chunk (the document routes among
-// them). The value is therefore loaded lazily inside `extractPdfText` via a
-// runtime `import()`, which lets pdf-parse initialise pdfjs on its own terms and
-// never touches DOMMatrix at load — verified to extract a real text layer under
-// plain Node. Keep the surface type as a type-only import (fully erased at
-// build, no runtime module evaluation).
-import type { PDFParse } from "pdf-parse";
-
 import { annotate } from "@/lib/logging/context";
 
+// pdfjs' module top-level references the browser-only `DOMMatrix` global. A
+// STATIC import makes the Turbopack server chunk evaluate that reference at
+// instantiation and throw `DOMMatrix is not defined`, taking down every
+// route/worker that shares the chunk (the document routes among them). The
+// module is therefore loaded lazily inside `extractPdfText` via a runtime
+// `import()`, and only the slice of its API used here is typed, structurally.
+interface PdfTextItem {
+  str: string;
+  /** [a, b, c, d, x, y]: the run's text matrix; x and y place it on the page. */
+  transform: number[];
+  width: number;
+  height: number;
+  hasEOL: boolean;
+}
+interface PdfTextPage {
+  getTextContent(): Promise<{ items: Array<PdfTextItem | object> }>;
+  cleanup(): void;
+}
+interface PdfTextDocument {
+  numPages: number;
+  getPage(pageNumber: number): Promise<PdfTextPage>;
+}
+interface PdfTextLoadingTask {
+  promise: Promise<PdfTextDocument>;
+  destroy(): Promise<void>;
+}
+interface PdfTextModule {
+  getDocument(opts: {
+    data: Uint8Array;
+    verbosity?: number;
+    isEvalSupported?: boolean;
+    useSystemFonts?: boolean;
+  }): PdfTextLoadingTask;
+}
+
+/**
+ * One page's text runs as text: a new line where a run ends its line or the
+ * next one sits on another, a space where two runs on one line leave a gap,
+ * nothing where they touch (a word split across two fonts stays one word).
+ */
+function pageText(items: Array<PdfTextItem | object>): string {
+  let text = "";
+  let last: { end: number; y: number; size: number } | null = null;
+  for (const item of items) {
+    // Marked-content boundaries carry no text.
+    if (!("str" in item)) continue;
+    const x = item.transform[4] ?? 0;
+    const y = item.transform[5] ?? 0;
+    const size = Math.abs(item.transform[3] ?? 0) || item.height || 1;
+    if (last && !text.endsWith("\n")) {
+      if (Math.abs(y - last.y) > last.size / 2) text += "\n";
+      else if (x - last.end > size * 0.1 && !/\s$/.test(text)) text += " ";
+    }
+    text += item.str;
+    if (item.hasEOL) text += "\n";
+    last = { end: x + item.width, y, size };
+  }
+  return text;
+}
+
 /**
  * Minimum joined text length for a PDF to count as a real text-layer document.
  * Below this a PDF is treated as scanned/empty — a few stray glyphs from a form
@@ -68,19 +125,31 @@ export type LocalExtractResult =
 export async function extractPdfText(
   buffer: Buffer,
 ): Promise<LocalExtractResult> {
-  let parser: PDFParse | null = null;
+  let task: PdfTextLoadingTask | null = null;
   try {
     // Lazy load: keeps pdfjs/DOMMatrix out of the eager server chunk (see the
     // import-site note). Resolved at first PDF extraction, then module-cached.
-    const { PDFParse } = await import("pdf-parse");
-    parser = new PDFParse({ data: new Uint8Array(buffer) });
-    const result = await parser.getText({
-      // Bound the page walk and drop the default "-- N of M --" page markers so
-      // only real document text is tokenised.
-      first: LOCAL_MAX_PAGES,
-      pageJoiner: "",
+    const pdfjs =
+      (await import("pdfjs-dist/legacy/build/pdf.mjs")) as unknown as PdfTextModule;
+    task = pdfjs.getDocument({
+      data: new Uint8Array(buffer),
+      verbosity: 0, // VerbosityLevel.ERRORS — no console spam on odd PDFs.
+      isEvalSupported: false, // never eval font programs (defence in depth).
+      useSystemFonts: false,
     });
-    const text = (result.text ?? "").trim();
+    const doc = await task.promise;
+    // Bound the page walk so a pathological many-page PDF costs no more.
+    const pageCount = Math.min(doc.numPages, LOCAL_MAX_PAGES);
+    const pages: string[] = [];
+    for (let n = 1; n <= pageCount; n++) {
+      const page = await doc.getPage(n);
+      try {
+        pages.push(pageText((await page.getTextContent()).items));
+      } finally {
+        page.cleanup();
+      }
+    }
+    const text = pages.join("\n").trim();
     if (text.length < LOCAL_TEXT_MIN_CHARS) {
       return { ok: false, reason: "empty" };
     }
@@ -88,9 +157,9 @@ export async function extractPdfText(
   } catch {
     return { ok: false, reason: "error" };
   } finally {
-    if (parser) {
+    if (task) {
       try {
-        await parser.destroy();
+        await task.destroy();
       } catch {
         // Best-effort teardown; a destroy failure must not mask the result.
       }