Merge remote-tracking branch 'origin/release/v1.46.0' into feat/documents-recently-deleted

src/__tests__/single-pdfjs-guard.test.ts

@@ -5,16 +5,18 @@
  *
  * Under Node, pdf.js keeps its in-process worker on `globalThis.pdfjsWorker`,
  * and whichever copy opens a document first owns it until the process exits.
- * The local text read ran a library that pinned its own, older pdfjs-dist,
- * while the rasterizer ran the app's. Whichever ran first broke the other
- * with "The API version … does not match the Worker version …": after a text
- * read every thumbnail and every vision read of a scan failed, after a render
- * every local text read failed without a word, until a restart. Nothing in
- * the gate noticed, because each path passed on its own and the image tracing
- * shipped every `pdfjs-dist@*` the store held.
+ * Local text extraction went through `pdf-parse`, which pinned its own, older
+ * pdfjs-dist, while the rasterizer ran the app's. Whichever ran first broke
+ * the other with "The API version "6.4.299" does not match the Worker version
+ * "5.4.296"": after a text extraction every thumbnail and every vision read
+ * of a scan failed, after a raster every local text extraction failed without
+ * a word, until a restart. Nothing in the gate noticed, because each path
+ * passed on its own and the image tracing shipped every `pdfjs-dist@*` the
+ * store held.
  *
  * ## What it proves and what it does not
  *
+ *   - package.json pins pdfjs-dist exactly and lists no pdf-parse.
  *   - The lockfile resolves exactly one pdfjs-dist, the version package.json
  *     pins, and no package in it depends on pdfjs-dist except the app itself.
  *     A new dependency that brings its own pdf.js, even the same version,
@@ -22,21 +24,28 @@
  *     names pdfjs-dist is either understood or reported.
  *   - The checkout resolves that same version from the working directory,
  *     which is how the server finds it.
- *   - No production module under `src/` imports pdfjs-dist except
- *     `src/lib/documents/pdfjs.ts`, so both paths open documents with the same
- *     options, and nothing imports pdf-parse.
+ *   - No runtime module under `src/` imports pdf-parse, and none imports
+ *     pdfjs-dist (a type-only import included) except
+ *     `src/lib/documents/pdfjs.ts`, so the text read and the rasterizer open
+ *     documents with the same options. Comments are stripped first, so prose
+ *     that names a package does not count.
  *
  * It does not prove the image carries one copy; the Dockerfile hoists the
  * pinned version by name and the tracing glob can only find what the lockfile
- * installed. The behaviour itself is pinned by the two
- * `pdfjs-one-runtime-*` tests under `src/lib/documents/__tests__/`.
+ * installed. The behaviour itself is pinned by the two `pdfjs-one-version-*`
+ * tests under `src/lib/documents/__tests__/`, one per order, each in a
+ * process of its own.
  */
 import { createRequire } from "node:module";
 import { join } from "node:path";
 
 import { describe, expect, it } from "vitest";
 
-import { readSourceFolder, readSourceText } from "./helpers/source-files";
+import {
+  readSourceFolder,
+  readSourceText,
+  stripComments,
+} from "./helpers/source-files";
 
 const ROOT = process.cwd();
 /** Relative to `src/`. */
@@ -84,27 +93,50 @@ function readLockfile(lockfile: string): {
   return { versions, rootImporters, unexplained };
 }
 
-/** Production modules under `src/` that import a pdf.js package. */
-function pdfjsImporters(files: Map<string, string>): string[] {
-  const specifier =
-    /(?:\bfrom\s*|\bimport\s*\(\s*|\brequire\s*\(\s*|^\s*import\s*)["'](?:pdfjs-dist|pdf-parse)(?:\/[^"']*)?["']/m;
+/** A static import, a re-export, a dynamic import or a require of `name`. */
+function importOf(name: string): RegExp {
+  return new RegExp(
+    String.raw`(?:\bfrom\s*|\bimport\s*\(\s*|\brequire\s*\(\s*|\bimport\s+)["']` +
+      name +
+      String.raw`(?:/[^"']*)?["']`,
+  );
+}
+const PDFJS_IMPORT = importOf("pdfjs-dist");
+const PDF_PARSE_IMPORT = importOf("pdf-parse");
+
+/** The modules whose comment-free source matches `pattern`, sorted. */
+function importers(files: Map<string, string>, pattern: RegExp): string[] {
   return [...files]
-    .filter(([, source]) => specifier.test(source))
+    .filter(([, source]) => pattern.test(source))
     .map(([path]) => path)
     .sort();
 }
 
-/** Production modules under `src/`, keyed by their path relative to it. */
-function productionSources(): Map<string, string> {
+/**
+ * Runtime modules under `src/`, keyed by their path relative to it, with
+ * comments stripped. The suites, mocks and generated client are not loaded
+ * by the server.
+ */
+function runtimeSources(): Map<string, string> {
   return new Map(
     readSourceFolder("src", {
       floor: 1000,
-      extensions: [".ts", ".tsx", ".mts", ".js", ".mjs"],
+      extensions: [
+        ".ts",
+        ".tsx",
+        ".mts",
+        ".cts",
+        ".js",
+        ".jsx",
+        ".mjs",
+        ".cjs",
+      ],
       include: (rel) =>
         !rel.startsWith("generated/") &&
-        !rel.includes("__tests__/") &&
+        !rel.split("/").includes("__tests__") &&
+        !rel.split("/").includes("__mocks__") &&
         !/\.(?:test|spec)\.[mc]?[tj]sx?$/.test(rel),
-    }),
+    }).map(([rel, source]) => [rel, stripComments(source)]),
   );
 }
 
@@ -132,10 +164,11 @@ describe("one pdf.js (#1277)", () => {
     expect(resolved.version).toBe(pinned);
   });
 
-  it("only the shared loader imports pdf.js", () => {
-    const sources = productionSources();
+  it("no runtime module imports pdf-parse, and only the shared loader imports pdf.js", () => {
+    const sources = runtimeSources();
     expect(sources.has(LOADER), `${LOADER} exists`).toBe(true);
-    expect(pdfjsImporters(sources)).toEqual([LOADER]);
+    expect(importers(sources, PDF_PARSE_IMPORT)).toEqual([]);
+    expect(importers(sources, PDFJS_IMPORT)).toEqual([LOADER]);
   });
 
   it("the readers catch what they exist to catch", () => {
@@ -158,15 +191,32 @@ describe("one pdf.js (#1277)", () => {
     expect([...lock.versions].sort()).toEqual(["5.4.296", "6.4.299"]);
     expect(lock.unexplained).toEqual(["13: pdfjs-dist: 5.4.296"]);
 
-    const importers = pdfjsImporters(
-      new Map([
+    for (const line of [
+      'import { PDFParse } from "pdf-parse";',
+      'import type { PDFParse } from "pdf-parse";',
+      'const { PDFParse } = await import("pdf-parse");',
+      "const m = require('pdf-parse/worker');",
+      'export { PDFParse } from "pdf-parse";',
+    ]) {
+      expect(PDF_PARSE_IMPORT.test(line), line).toBe(true);
+    }
+    expect(PDF_PARSE_IMPORT.test('import x from "pdf-parser";')).toBe(false);
+
+    const sources = new Map(
+      [
         [LOADER, 'await import("pdfjs-dist/legacy/build/pdf.mjs")'],
-        ["a", 'import { PDFParse } from "pdf-parse";'],
+        [
+          "a",
+          'import type { TextItem } from "pdfjs-dist/types/src/display/api";',
+        ],
         ["b", "const m = await import(\n  'pdfjs-dist'\n);"],
         ["c", 'import "pdfjs-dist/legacy/build/pdf.worker.mjs";'],
-        ["d", '// mentions "pdfjs-dist" in prose only'],
-      ]),
+        ["d", '// once read with import("pdfjs-dist") here'],
+        ["e", 'const name = "pdfjs-dist";'],
+      ].map(([rel, source]) => [rel!, stripComments(source!)]),
+    );
+    expect(importers(sources, PDFJS_IMPORT)).toEqual(
+      ["a", "b", "c", LOADER].sort(),
     );
-    expect(importers).toEqual(["a", "b", "c", LOADER].sort());
   });
 });

src/lib/documents/__tests__/pdfjs-one-runtime-raster-first.test.ts

@@ -1,26 +0,0 @@
-import { describe, it, vi } from "vitest";
-
-/**
- * GitHub #1277, rasterization first: a render used to claim the in-process
- * pdf.js worker for its own copy, and every later local text read failed
- * without a word, leaving the document out of the search index. See
- * `pdfjs-one-runtime.ts`.
- */
-
-vi.mock("@/lib/logging/context", () => ({ annotate: vi.fn() }));
-vi.mock("@/lib/logging/signal", () => ({ logCaught: vi.fn() }));
-
-import {
-  expectFreshProcess,
-  expectRaster,
-  expectTextRead,
-} from "./pdfjs-one-runtime";
-
-describe("one pdf.js per process (#1277): rasterization first", () => {
-  it("reads text after a rasterization, and rasterizes again after that", async () => {
-    expectFreshProcess();
-    await expectRaster();
-    await expectTextRead();
-    await expectRaster();
-  });
-});

src/lib/documents/__tests__/pdfjs-one-runtime-text-first.test.ts

@@ -1,26 +0,0 @@
-import { describe, it, vi } from "vitest";
-
-/**
- * GitHub #1277, text first: a local text read used to claim the in-process
- * pdf.js worker for its own copy, and every later rasterization (thumbnails,
- * the vision read of a scan) failed until a restart. See
- * `pdfjs-one-runtime.ts`.
- */
-
-vi.mock("@/lib/logging/context", () => ({ annotate: vi.fn() }));
-vi.mock("@/lib/logging/signal", () => ({ logCaught: vi.fn() }));
-
-import {
-  expectFreshProcess,
-  expectRaster,
-  expectTextRead,
-} from "./pdfjs-one-runtime";
-
-describe("one pdf.js per process (#1277): text read first", () => {
-  it("rasterizes after a text read, and reads text again after that", async () => {
-    expectFreshProcess();
-    await expectTextRead();
-    await expectRaster();
-    await expectTextRead();
-  });
-});

src/lib/documents/__tests__/pdfjs-one-runtime.ts

@@ -1,50 +0,0 @@
-/**
- * Shared steps for the two `pdfjs-one-runtime-*` files (GitHub #1277).
- *
- * Under Node, pdf.js runs its worker in-process and keeps it on
- * `globalThis.pdfjsWorker`. Whichever pdf.js copy opens a document first
- * owns that global for the life of the process, and a different copy then
- * refuses every document with "The API version … does not match the Worker
- * version …". The text reader and the rasterizer once loaded different
- * copies, so whichever ran first broke the other until a restart.
- *
- * Each order needs a fresh process, because the global cannot be taken back
- * once claimed; that is why the two orders live in two files (the isolated
- * project gives each file its own process) instead of two tests here.
- */
-import { readFileSync } from "node:fs";
-import { join } from "node:path";
-
-import { expect } from "vitest";
-
-import { extractPdfText } from "../local-extract";
-import { rasterizePdf } from "../rasterize-pdf";
-
-const FIXTURE = join(
-  process.cwd(),
-  "tests",
-  "fixtures",
-  "pdf",
-  "fallback-fonts.pdf",
-);
-
-const pdf = () => readFileSync(FIXTURE);
-
-/** Nothing in this process has opened a PDF yet. */
-export function expectFreshProcess(): void {
-  expect(
-    (globalThis as { pdfjsWorker?: unknown }).pdfjsWorker,
-    "the file must run in its own process; another file already opened a PDF",
-  ).toBeUndefined();
-}
-
-export async function expectTextRead(): Promise<void> {
-  const result = await extractPdfText(pdf());
-  expect(result).toMatchObject({ ok: true, source: "local-pdf" });
-  if (result.ok) expect(result.text).toContain("Test Insurance AG");
-}
-
-export async function expectRaster(): Promise<void> {
-  const result = await rasterizePdf(pdf(), 1);
-  expect(result).toMatchObject({ ok: true });
-}

src/lib/documents/__tests__/pdfjs-one-version-extract-first.test.ts

@@ -0,0 +1,50 @@
+import { readFileSync } from "node:fs";
+import { join } from "node:path";
+
+import { describe, expect, it, vi } from "vitest";
+
+/**
+ * Text extraction and rasterising run in one process. In Node, pdfjs keeps its
+ * worker in one global, `globalThis.pdfjsWorker`. When `pdf-parse` brought its
+ * own pdfjs (5.4.296) beside the app's (6.4.299), the copy that ran first set
+ * that global, and the other copy refused it: "The API version "6.4.299" does
+ * not match the Worker version "5.4.296"". Thumbnails then failed as
+ * `raster-failed`, vision reads of a PDF answered 422, and the local text index
+ * stayed empty with no log line. The fix: read the text with the app's own
+ * pdfjs.
+ *
+ * This file extracts text first and rasterises second. Its twin in this
+ * folder runs the reverse order. Each order needs its own file, because a
+ * process keeps the first worker it loads.
+ *
+ * Mutation check: read the text through `pdf-parse` again in
+ * `extractPdfText` → the raster step here fails.
+ */
+vi.mock("@/lib/logging/context", () => ({ annotate: vi.fn() }));
+vi.mock("@/lib/logging/signal", () => ({ logCaught: vi.fn() }));
+
+import { extractPdfText } from "../local-extract";
+import { rasterizePdf } from "../rasterize-pdf";
+
+const fixture = readFileSync(
+  join(process.cwd(), "tests", "fixtures", "pdf", "fallback-fonts.pdf"),
+);
+
+describe("one pdfjs for text and raster: text first", () => {
+  it("extracts, rasterises and extracts again in one process", async () => {
+    // The order only means something in a process that has not opened a PDF.
+    expect(
+      (globalThis as { pdfjsWorker?: unknown }).pdfjsWorker,
+      "this file must run in a process of its own",
+    ).toBeUndefined();
+
+    const first = await extractPdfText(fixture);
+    expect(first.ok).toBe(true);
+
+    const raster = await rasterizePdf(fixture, 1);
+    expect(raster.ok).toBe(true);
+
+    const again = await extractPdfText(fixture);
+    expect(again).toEqual(first);
+  });
+});

src/lib/documents/__tests__/pdfjs-one-version-raster-first.test.ts

@@ -0,0 +1,50 @@
+import { readFileSync } from "node:fs";
+import { join } from "node:path";
+
+import { describe, expect, it, vi } from "vitest";
+
+/**
+ * Text extraction and rasterising run in one process. In Node, pdfjs keeps its
+ * worker in one global, `globalThis.pdfjsWorker`. When `pdf-parse` brought its
+ * own pdfjs (5.4.296) beside the app's (6.4.299), the copy that ran first set
+ * that global, and the other copy refused it: "The API version "6.4.299" does
+ * not match the Worker version "5.4.296"". Thumbnails then failed as
+ * `raster-failed`, vision reads of a PDF answered 422, and the local text index
+ * stayed empty with no log line. The fix: read the text with the app's own
+ * pdfjs.
+ *
+ * This file rasterises first and extracts text second. Its twin in this
+ * folder runs the reverse order. Each order needs its own file, because a
+ * process keeps the first worker it loads.
+ *
+ * Mutation check: read the text through `pdf-parse` again in
+ * `extractPdfText` → the text step here fails.
+ */
+vi.mock("@/lib/logging/context", () => ({ annotate: vi.fn() }));
+vi.mock("@/lib/logging/signal", () => ({ logCaught: vi.fn() }));
+
+import { extractPdfText } from "../local-extract";
+import { rasterizePdf } from "../rasterize-pdf";
+
+const fixture = readFileSync(
+  join(process.cwd(), "tests", "fixtures", "pdf", "fallback-fonts.pdf"),
+);
+
+describe("one pdfjs for text and raster: raster first", () => {
+  it("rasterises, extracts and rasterises again in one process", async () => {
+    // The order only means something in a process that has not opened a PDF.
+    expect(
+      (globalThis as { pdfjsWorker?: unknown }).pdfjsWorker,
+      "this file must run in a process of its own",
+    ).toBeUndefined();
+
+    const first = await rasterizePdf(fixture, 1);
+    expect(first.ok).toBe(true);
+
+    const text = await extractPdfText(fixture);
+    expect(text.ok).toBe(true);
+
+    const again = await rasterizePdf(fixture, 1);
+    expect(again.ok).toBe(true);
+  });
+});

src/lib/documents/__tests__/pdfjs-text-layer.test.ts

@@ -1,80 +1,89 @@
 import { jsPDF } from "jspdf";
+import type {
+  TextItem,
+  TextMarkedContent,
+} from "pdfjs-dist/types/src/display/api";
 import { describe, expect, it } from "vitest";
 
-import { pageTextLayer, readPdfText, type PdfTextItem } from "../pdfjs";
+import { joinTextRuns, readPdfText } from "../pdfjs";
 
 /**
- * The text layer as the content index reads it. The layout is the one the
- * local text read produced before it moved onto the app's own pdf.js (#1277):
- * a break where pdf.js marks one or the next item sits on a lower line, a tab
- * between far-apart items on one line, each page followed by a blank line.
+ * The text layer as the content index reads it (#1277): runs in order, a line
+ * break where a run ends its line or the next one sits on another, a space
+ * where two runs on one line leave a gap, and nothing where they touch, so a
+ * word drawn in two fonts stays one word and a search for it still finds it.
  */
 
-/** A page whose viewport maps PDF space to itself, flipped like pdf.js. */
-function pageOf(items: PdfTextItem[]) {
-  return {
-    getViewport: () => ({
-      width: 600,
-      height: 800,
-      convertToViewportPoint: (x: number, y: number) => [x, 800 - y],
-    }),
-    getTextContent: async () => ({ items }),
-  };
-}
-
-function item(
+/** A 10 pt run starting at (x, y), 5 pt per character. */
+function run(
   str: string,
   x: number,
   y: number,
-  extra: Partial<PdfTextItem> = {},
-): PdfTextItem {
+  extra: Partial<TextItem> = {},
+): TextItem {
   return {
     str,
+    dir: "ltr",
     transform: [10, 0, 0, 10, x, y],
     width: str.length * 5,
     height: 10,
+    fontName: "g_d0_f1",
     hasEOL: false,
     ...extra,
   };
 }
 
-describe("pageTextLayer", () => {
-  it("breaks once where pdf.js marks the end of a line", async () => {
-    const text = await pageTextLayer(
-      pageOf([
-        item("Haemoglobin", 10, 700, { hasEOL: true }),
-        item("14.2 g/dL", 10, 688),
+describe("joinTextRuns", () => {
+  it("keeps a word drawn in two fonts whole", () => {
+    expect(
+      joinTextRuns([
+        run("Haem", 10, 700),
+        run("oglobin", 30, 700, { fontName: "g_d0_f2" }),
       ]),
-    );
-    expect(text).toBe("Haemoglobin\n14.2 g/dL");
+    ).toBe("Haemoglobin");
   });
 
-  it("breaks when the next item sits a line lower without a marked end", async () => {
-    const text = await pageTextLayer(
-      pageOf([item("Creatinine", 10, 700), item("0.9 mg/dL", 10, 680)]),
+  it("puts a space between runs on one line that leave a gap", () => {
+    expect(joinTextRuns([run("Glucose", 10, 700), run("92", 200, 700)])).toBe(
+      "Glucose 92",
     );
-    expect(text).toBe("Creatinine\n0.9 mg/dL");
   });
 
-  it("does not break for a small baseline shift on one line", async () => {
-    const text = await pageTextLayer(
-      pageOf([item("10", 10, 700), item("3", 21, 703)]),
-    );
-    expect(text).toBe("103");
+  it("breaks once where a run ends its line", () => {
+    expect(
+      joinTextRuns([
+        run("Haemoglobin", 10, 700, { hasEOL: true }),
+        run("14.2 g/dL", 10, 688),
+      ]),
+    ).toBe("Haemoglobin\n14.2 g/dL");
   });
 
-  it("separates far-apart items on one line with a tab", async () => {
-    const text = await pageTextLayer(
-      pageOf([item("Glucose", 10, 700), item("92", 200, 700)]),
-    );
-    expect(text).toBe("Glucose\t92");
+  it("breaks where the next run sits on another line without a marked end", () => {
+    expect(
+      joinTextRuns([run("Creatinine", 10, 700), run("0.9", 10, 680)]),
+    ).toBe("Creatinine\n0.9");
   });
 
-  it("skips marked-content items, which carry no text", async () => {
-    const text = await pageTextLayer(
-      pageOf([{ hasEOL: false } as PdfTextItem, item("Text", 10, 700)]),
-    );
-    expect(text).toBe("Text");
+  it("keeps a raised run on its line", () => {
+    expect(joinTextRuns([run("10", 10, 700), run("9", 20, 703)])).toBe("109");
+  });
+
+  it("adds no blank line for an empty run that only ends a line", () => {
+    expect(
+      joinTextRuns([
+        run("Line", 10, 700, { hasEOL: true }),
+        run("", 30, 700, { hasEOL: true }),
+        run("Next", 10, 688),
+      ]),
+    ).toBe("Line\nNext");
+  });
+
+  it("skips marked-content markers, which carry no text", () => {
+    const marker: TextMarkedContent = {
+      type: "beginMarkedContent",
+      id: "mc0",
+    };
+    expect(joinTextRuns([marker, run("Text", 10, 700)])).toBe("Text");
   });
 });
 
@@ -89,10 +98,10 @@ describe("readPdfText", () => {
     return Buffer.from(doc.output("arraybuffer"));
   }
 
-  it("joins pages with a blank line and adds no page markers", async () => {
+  it("reads the pages one after another and adds no page markers", async () => {
     const text = await readPdfText(pages(2));
     expect(text).toBe(
-      "Page 1 lab report\nsecond line\n\nPage 2 lab report\nsecond line\n\n",
+      "Page 1 lab report\nsecond line\nPage 2 lab report\nsecond line",
     );
   });
 
@@ -102,6 +111,17 @@ describe("readPdfText", () => {
     expect(text).not.toContain("Page 3");
   });
 
+  it("keeps a word whole that a real PDF draws in two fonts", async () => {
+    const doc = new jsPDF();
+    doc.setFont("helvetica", "bold");
+    doc.text("Haem", 10, 10);
+    const joint = 10 + doc.getTextWidth("Haem");
+    doc.setFont("times", "normal");
+    doc.text("oglobin 14.2 g/dL", joint, 10);
+    const text = await readPdfText(Buffer.from(doc.output("arraybuffer")));
+    expect(text).toBe("Haemoglobin 14.2 g/dL");
+  });
+
   it("throws what pdf.js throws on a file that is not a PDF", async () => {
     await expect(readPdfText(Buffer.from("not a pdf"))).rejects.toThrow();
   });

src/lib/documents/content-index.ts

@@ -325,7 +325,7 @@ export async function loadDocumentChatText(
  *   - `vision`    → an AI provider transcribed the stored original (image or PDF,
  *                   incl. scanned) — the AI-first primary path.
  *   - `text-ocr`  → browser-OCR text posted by the client (opt-in local OCR).
- *   - `local-pdf` → server-side text-layer extraction of a PDF (pdf.js), no
+ *   - `local-pdf` → server-side text-layer extraction of a PDF (pdfjs), no
  *                   provider, no egress — the fallback when no provider is usable.
  *   - `local-ocr` → server-side OCR of a scanned image/PDF (deferred follow-up).
  */

src/lib/documents/index-document.ts

@@ -24,7 +24,7 @@
  *      configured, the toggle is OFF for an external pick, the provider can't
  *      read this file (e.g. a text-layer PDF on a non-Anthropic account), the
  *      daily budget is reached, or the provider call fails — fall back to
- *      server-side text-layer extraction (pdf.js, no egress, milliseconds).
+ *      server-side text-layer extraction (pdfjs, no egress, milliseconds).
  *      So content search works at a baseline even with no AI, and every
  *      text-layer PDF is searchable regardless of provider.
  *

src/lib/documents/local-extract.ts

@@ -3,9 +3,18 @@
  *
  * When NO AI provider is configured — or the user has not consented to one —
  * content search still needs a baseline. This module extracts a PDF's embedded
- * text layer SERVER-SIDE through the app's one pdf.js (`./pdfjs`; reading the
- * text layer never draws, so the native canvas does no work here), in
- * milliseconds, with no OCR and no third-party egress.
+ * text layer SERVER-SIDE with pdfjs' `getTextContent` (the pure-JS/wasm core,
+ * never the native `@napi-rs/canvas`), in milliseconds, with no OCR and no
+ * third-party egress.
+ *
+ * It reads through `./pdfjs`, the one loader `./rasterize-pdf` opens documents
+ * with too, so both run the app's own `pdfjs-dist` with the same options. It
+ * used to go through `pdf-parse`, which brings a second pdfjs of its own, and
+ * the two cannot share a process: both take their worker from
+ * `globalThis.pdfjsWorker`, so whichever ran first broke the other ("The API
+ * version "6.4.299" does not match the Worker version "5.4.296""). Thumbnails
+ * and vision reads failed after a text extraction, and the text extraction
+ * failed after a raster.
  * The extracted text feeds the SAME blind, encrypted content index the provider
  * path writes (`upsertContentIndex`), so nothing readable is stored at rest.
  *
@@ -62,11 +71,10 @@ export type LocalExtractResult =
 export async function extractPdfText(
   buffer: Buffer,
 ): Promise<LocalExtractResult> {
-  let raw: string;
+  let text: string;
   try {
-    // Bounded page walk; pages joined by a blank line, with no page markers,
-    // so only real document text is tokenised.
-    raw = await readPdfText(buffer, LOCAL_MAX_PAGES);
+    // Bound the page walk so a pathological many-page PDF costs no more.
+    text = (await readPdfText(buffer, LOCAL_MAX_PAGES)).trim();
   } catch (err) {
     const errorType = err instanceof Error ? err.name || "Error" : typeof err;
     annotate({
@@ -78,7 +86,6 @@ export async function extractPdfText(
     });
     return { ok: false, reason: "error" };
   }
-  const text = raw.trim();
   if (text.length < LOCAL_TEXT_MIN_CHARS) {
     return { ok: false, reason: "empty" };
   }

src/lib/documents/pdfjs.ts

@@ -1,5 +1,6 @@
 /**
- * The one pdf.js this server runs, and the one way it is opened.
+ * The one pdf.js this server runs, the one way it is opened, and the one way
+ * its text layer is read.
  *
  * Under Node, pdf.js does not start a worker thread: it loads its worker code
  * into the same process and keeps it on `globalThis.pdfjsWorker`. Whichever
@@ -15,7 +16,9 @@
  * reads the browser-only `DOMMatrix` global, so a static import would make the
  * server chunk evaluate it at instantiation and throw `DOMMatrix is not
  * defined`, taking down every route and worker sharing the chunk. The import
- * resolves once and is module-cached after that.
+ * resolves once and is module-cached after that. The type imports below are
+ * erased at build, so they load nothing; they let the compiler check every
+ * call here against pdf.js' own API.
  *
  * This module has no logging of its own, so the end-to-end suite can read
  * report text with it; the callers say what failed.
@@ -25,74 +28,11 @@ import { existsSync } from "node:fs";
 import { createRequire } from "node:module";
 import { dirname, join } from "node:path";
 
-// Structural types for the slice of the pdf.js API used here and in the
-// rasterizer. Type-only, so nothing evaluates pdf.js for its types.
-export interface PdfViewport {
-  width: number;
-  height: number;
-  convertToViewportPoint(x: number, y: number): number[];
-}
-export interface PdfCanvas {
-  width: number;
-  height: number;
-  toBuffer(mime: "image/jpeg", quality?: number): Buffer;
-}
-export interface PdfContext2D {
-  getImageData(
-    x: number,
-    y: number,
-    width: number,
-    height: number,
-  ): { data: Uint8ClampedArray };
-}
-interface PdfCanvasAndContext {
-  canvas: PdfCanvas;
-  context: PdfContext2D;
-}
-interface PdfCanvasFactory {
-  create(width: number, height: number): PdfCanvasAndContext;
-  destroy?(target: PdfCanvasAndContext): void;
-}
-/** A text-layer item; marked-content items carry no `str` and are skipped. */
-export interface PdfTextItem {
-  str?: string;
-  hasEOL?: boolean;
-  transform?: number[];
-  width?: number;
-  height?: number;
-}
-export interface PdfPage {
-  getViewport(opts: { scale: number }): PdfViewport;
-  render(opts: {
-    canvasContext: unknown;
-    viewport: PdfViewport;
-    canvas: unknown;
-  }): { promise: Promise<void> };
-  cleanup(): void;
-  getTextContent(): Promise<{ items: PdfTextItem[] }>;
-}
-export interface PdfDocument {
-  numPages: number;
-  canvasFactory: PdfCanvasFactory;
-  getPage(pageNumber: number): Promise<PdfPage>;
-}
-export interface PdfLoadingTask {
-  promise: Promise<PdfDocument>;
-  destroy(): Promise<void>;
-}
-interface PdfDocumentOptions {
-  data: Uint8Array;
-  verbosity?: number;
-  isEvalSupported?: boolean;
-  useSystemFonts?: boolean;
-  standardFontDataUrl?: string;
-  cMapUrl?: string;
-  cMapPacked?: boolean;
-  wasmUrl?: string;
-}
-interface PdfjsModule {
-  getDocument(opts: PdfDocumentOptions): PdfLoadingTask;
-}
+import type { PDFDocumentLoadingTask } from "pdfjs-dist";
+import type {
+  TextItem,
+  TextMarkedContent,
+} from "pdfjs-dist/types/src/display/api";
 
 /**
  * Where pdf.js finds the data it does not carry in its JS: the standard 14
@@ -169,15 +109,13 @@ export function __resetPdfjsAssetsForTests(): void {
  */
 export async function openPdfDocument(
   data: Uint8Array | Buffer,
-): Promise<PdfLoadingTask> {
-  const pdfjs =
-    (await import("pdfjs-dist/legacy/build/pdf.mjs")) as unknown as PdfjsModule;
+): Promise<PDFDocumentLoadingTask> {
+  const pdfjs = await import("pdfjs-dist/legacy/build/pdf.mjs");
   const assets = resolvePdfjsAssets();
   return pdfjs.getDocument({
     // A copy: pdf.js transfers the buffer it is given to its worker.
     data: new Uint8Array(data),
     verbosity: 0, // VerbosityLevel.ERRORS: no console noise on odd PDFs.
-    isEvalSupported: false, // never eval font programs (defence in depth).
     // There are no system fonts to use: substitutes come from pdf.js' own
     // standard font data (and, when rendering, the registered fallbacks).
     useSystemFonts: false,
@@ -192,73 +130,83 @@ export async function openPdfDocument(
   });
 }
 
-/** Vertical gap (pt) past which the next item starts a new line. */
-const LINE_THRESHOLD = 4.6;
-/** Horizontal gap (pt) on one line past which the next item is a new cell. */
-const CELL_THRESHOLD = 7;
-
 /**
- * One page's text layer as plain text: items in content order, a line break
- * where pdf.js marks one or where the next item sits on a lower line, and a
- * tab between items on one line that are far apart (table cells).
- *
- * This is the layout `pdf-parse` produced with its defaults, kept so that
- * swapping it out did not change what the content index sees.
+ * Where a text run sits on the page. pdfjs gives each run a text matrix
+ * `[a, b, c, d, x, y]`: `x` and `y` are where the run starts, and `d` is its
+ * vertical scale, which for upright text is the font size.
  */
-export async function pageTextLayer(
-  page: Pick<PdfPage, "getViewport" | "getTextContent">,
-): Promise<string> {
-  const viewport = page.getViewport({ scale: 1 });
-  const { items } = await page.getTextContent();
-  const parts: string[] = [];
-  let lastX: number | undefined;
-  let lastY: number | undefined;
-  let lineHeight = 0;
-  for (const item of items) {
-    if (typeof item.str !== "string") continue;
-    const transform = item.transform ?? [1, 0, 0, 1, 0, 0];
-    const [x = 0, y = 0] = viewport.convertToViewportPoint(
-      transform[4] ?? 0,
-      transform[5] ?? 0,
-    );
-    let str = item.str;
-    if (lastY !== undefined) {
-      const gap = Math.abs(lastY - y);
-      if (gap > LINE_THRESHOLD) {
-        const previous = parts.at(-1);
-        const opensLine =
-          str.startsWith("\n") || (str.trim() === "" && item.hasEOL === true);
-        if (
-          previous !== undefined &&
-          !previous.endsWith("\n") &&
-          !opensLine &&
-          gap - 1 > lineHeight
-        ) {
-          parts.push("\n");
-          lineHeight = 0;
-        }
-      } else if (
-        gap < LINE_THRESHOLD &&
-        lastX !== undefined &&
-        Math.abs(lastX - x) > CELL_THRESHOLD
-      ) {
-        str = `\t${str}`;
-      }
-    }
-    parts.push(str);
-    lastX = x + (item.width ?? 0);
-    lastY = y;
-    lineHeight = Math.max(lineHeight, item.height ?? 0);
-    if (item.hasEOL) parts.push("\n");
-    if (item.hasEOL || str.endsWith("\n")) lineHeight = 0;
-  }
-  return parts.join("");
+interface RunPosition {
+  x: number;
+  /** The baseline the run sits on. */
+  y: number;
+  /** Where the run ends: its start plus its width. */
+  end: number;
+  fontSize: number;
+}
+
+function positionOf(run: TextItem): RunPosition {
+  const [, , , scaleY, x = 0, y = 0] = run.transform as number[];
+  return {
+    x,
+    y,
+    end: x + run.width,
+    fontSize: Math.abs(scaleY ?? 0) || run.height || 1,
+  };
 }
 
 /**
- * The text layer of the first `maxPages` pages, each page followed by a blank
- * line. Throws what pdf.js throws (a malformed file, a password, a page that
- * will not parse); the document is torn down either way.
+ * The next run is on a new line when its baseline moved by more than half a
+ * font size. Runs on one line can sit a little higher or lower (a superscript,
+ * a mixed font), so a small shift does not count.
+ */
+function startsNewLine(previous: RunPosition, next: RunPosition): boolean {
+  return Math.abs(next.y - previous.y) > previous.fontSize / 2;
+}
+
+/**
+ * Two runs on one line are separate words when the gap between them is wider
+ * than a tenth of the font size. Runs that touch, such as a word drawn in two
+ * fonts, stay one word.
+ */
+function leavesWordGap(previous: RunPosition, next: RunPosition): boolean {
+  return next.x - previous.end > next.fontSize / 10;
+}
+
+/**
+ * One page's text: its runs in order, with a line break where a run ends a
+ * line or the next one starts a new line, and a space where two runs on one
+ * line leave a gap between words.
+ */
+export function joinTextRuns(
+  items: Array<TextItem | TextMarkedContent>,
+): string {
+  let text = "";
+  let previous: RunPosition | null = null;
+
+  for (const item of items) {
+    // Marked-content markers carry no text.
+    if (!("str" in item)) continue;
+    const position = positionOf(item);
+
+    if (previous && !text.endsWith("\n")) {
+      if (startsNewLine(previous, position)) {
+        text += "\n";
+      } else if (leavesWordGap(previous, position) && !/\s$/.test(text)) {
+        text += " ";
+      }
+    }
+    text += item.str;
+    if (item.hasEOL && !text.endsWith("\n")) text += "\n";
+
+    previous = position;
+  }
+  return text;
+}
+
+/**
+ * The text layer of the first `maxPages` pages, one page after another.
+ * Throws what pdf.js throws (a malformed file, a password, a page that will
+ * not parse); the document is torn down either way.
  */
 export async function readPdfText(
   data: Uint8Array | Buffer,
@@ -267,17 +215,17 @@ export async function readPdfText(
   const task = await openPdfDocument(data);
   try {
     const doc = await task.promise;
-    const last = Math.min(doc.numPages, maxPages);
-    let text = "";
-    for (let pageNumber = 1; pageNumber <= last; pageNumber++) {
-      const page = await doc.getPage(pageNumber);
+    const pageCount = Math.min(doc.numPages, maxPages);
+    const pages: string[] = [];
+    for (let n = 1; n <= pageCount; n++) {
+      const page = await doc.getPage(n);
       try {
-        text += `${await pageTextLayer(page)}\n\n`;
+        pages.push(joinTextRuns((await page.getTextContent()).items));
       } finally {
         page.cleanup();
       }
     }
-    return text;
+    return pages.join("\n");
   } finally {
     try {
       await task.destroy();

src/lib/documents/rasterize-pdf.ts

@@ -33,15 +33,7 @@ import { Buffer } from "node:buffer";
 import { annotate } from "@/lib/logging/context";
 
 import { nativeCanvasSupported } from "./native-canvas-support";
-import {
-  openPdfDocument,
-  resolvePdfjsAssets,
-  type PdfContext2D,
-  type PdfDocument,
-  type PdfjsAssets,
-  type PdfLoadingTask,
-  type PdfPage,
-} from "./pdfjs";
+import { openPdfDocument, resolvePdfjsAssets, type PdfjsAssets } from "./pdfjs";
 
 /**
  * Whether the PDF rasterizer is available in this build. The native
@@ -114,6 +106,33 @@ export type RasterResult =
     }
   | { ok: false; reason: "unsupported" | "render-failed" };
 
+/**
+ * The canvas pdf.js draws into under Node is `@napi-rs/canvas`. pdf.js' own
+ * declarations are written for the DOM and type the document's canvas factory
+ * as a bare `object`, so the slice used here is named: a DOM-shaped canvas
+ * and 2D context, plus the JPEG encoder the native canvas adds.
+ */
+interface RenderTarget {
+  canvas: HTMLCanvasElement & {
+    toBuffer(mime: "image/jpeg", quality?: number): Buffer;
+  };
+  context: CanvasRenderingContext2D;
+}
+interface RenderTargetFactory {
+  create(width: number, height: number): RenderTarget;
+  destroy?(target: RenderTarget): void;
+}
+
+/**
+ * A page's text layer, as far as `pageText` reads it: text runs, and
+ * marked-content markers that carry only a `type` and no text.
+ */
+interface TextLayerPage {
+  getTextContent(): Promise<{
+    items: Array<{ str?: string; hasEOL?: boolean; type?: string }>;
+  }>;
+}
+
 interface GlobalFontsLike {
   registerFromPath(path: string, nameAlias?: string): unknown;
 }
@@ -144,7 +163,11 @@ async function registerFallbackFamilies(assets: PdfjsAssets): Promise<void> {
 }
 
 /** Share of dark pixels on a rendered page (luminance below half). */
-function inkRatio(context: PdfContext2D, width: number, height: number) {
+function inkRatio(
+  context: Pick<CanvasRenderingContext2D, "getImageData">,
+  width: number,
+  height: number,
+) {
   const { data } = context.getImageData(0, 0, width, height);
   let dark = 0;
   for (let i = 0; i < data.length; i += 4) {
@@ -188,9 +211,7 @@ function headThatFits(text: string, room: number): string {
  * `RASTER_PAGE_TEXT_MAX` it is cut (at a line, else a space, else hard) and
  * marked.
  */
-export async function pageText(
-  page: Pick<PdfPage, "getTextContent">,
-): Promise<string> {
+export async function pageText(page: TextLayerPage): Promise<string> {
   const max = RASTER_PAGE_TEXT_MAX;
   try {
     const { items } = await page.getTextContent();
@@ -248,8 +269,7 @@ export async function rasterizePdf(
   maxPages: number = RASTER_MAX_PAGES,
   opts: { withText?: boolean } = {},
 ): Promise<RasterResult> {
-  let doc: PdfDocument | null = null;
-  let task: PdfLoadingTask | null = null;
+  let task: Awaited<ReturnType<typeof openPdfDocument>> | null = null;
   // CPU-feature gate: pdfjs renders through @napi-rs/canvas (Skia), whose x64
   // build uses AVX2 — an unsupported CPU dies with an uncatchable SIGILL on
   // the first render. Degrade to the existing no-raster path instead (scanned
@@ -273,7 +293,7 @@ export async function rasterizePdf(
     }
 
     task = await openPdfDocument(buffer);
-    doc = await task.promise;
+    const doc = await task.promise;
 
     // Never exceed the standing DoS ceiling, even if a caller asks for more.
     const cap = Math.min(Math.max(1, maxPages), RASTER_HARD_MAX_PAGES);
@@ -292,7 +312,7 @@ export async function rasterizePdf(
           longEdge > 0 ? Math.min(1, RASTER_TARGET_LONG_EDGE / longEdge) : 1;
         const viewport = page.getViewport({ scale });
 
-        const factory = doc.canvasFactory;
+        const factory = doc.canvasFactory as RenderTargetFactory;
         const cc = factory.create(
           Math.ceil(viewport.width),
           Math.ceil(viewport.height),