fix(documents): index every page of a PDF read by its text layer

"Read with AI" and the index backfill wrote a text-layer read as a complete
`local-pdf` index although it held only the first ten pages (the render
cap), while the local text index reads up to 200. `prepareVisionInput` takes
the number of pages a text-layer read may take, and both index writers ask
for the local index's own cap, so the index is complete.

The `partial` flag on a text read had no reader and is gone; the
extraction and the lab scan report their missing pages themselves.

Refs #1311

src/lib/documents/__tests__/index-document.test.ts

@@ -255,7 +255,6 @@ describe("indexDocumentContent — local fallback path", () => {
       ok: true,
       mode: "text",
       text: "Haemoglobin 14.2 g/dl",
-      partial: false,
     });
 
     const outcome = await indexDocumentContent("user-1", "doc-1");

src/lib/documents/__tests__/unrenderable-pdf-text-layer.test.ts

@@ -125,19 +125,26 @@ describe("on a server that cannot render PDF pages", () => {
     it("reads a text-layer PDF as text for a provider that only takes images", async () => {
       stored(buildPdf([LAB_PAGE, LETTER_PAGE]));
       const vision = await prepareVisionInput(DOC, false);
-      expect(vision).toMatchObject({ ok: true, mode: "text", partial: false });
+      expect(vision).toMatchObject({ ok: true, mode: "text" });
       if (!vision.ok || vision.mode !== "text") return;
       expect(vision.text).toContain("Creatinine 0.9 mg/dl");
       expect(vision.text).toContain("community-acquired pneumonia");
     }, 30_000);
 
-    it("marks the text read partial when a page is a scan", async () => {
-      stored(buildPdf([LAB_PAGE, SCAN]));
-      await expect(prepareVisionInput(DOC, false)).resolves.toMatchObject({
-        ok: true,
-        mode: "text",
-        partial: true,
+    it("reads the render cap's pages by default, and as many as asked for the index", async () => {
+      const pages = Array.from({ length: 12 }, (_, i) => ({
+        text: `Page ${i + 1} creatinine 0.9 mg/dl reference 0.7 - 1.2`,
+      }));
+      stored(buildPdf(pages));
+      const capped = await prepareVisionInput(DOC, false);
+      const whole = await prepareVisionInput(DOC, false, {
+        textLayerPages: 200,
       });
+      if (!capped.ok || capped.mode !== "text") throw new Error("no text");
+      if (!whole.ok || whole.mode !== "text") throw new Error("no text");
+      expect(capped.text).toContain("Page 10 ");
+      expect(capped.text).not.toContain("Page 11 ");
+      expect(whole.text).toContain("Page 12 ");
     }, 30_000);
 
     it("answers a scan with pdfRendererUnavailable, never as a read", async () => {

src/lib/documents/ai-route-support.ts

@@ -13,7 +13,6 @@ import { Buffer } from "node:buffer";
 import { apiError } from "@/lib/api-response";
 import { prisma } from "@/lib/db";
 import {
-  isPartialTextLayer,
   readPdfTextLayer,
   textLayerText,
 } from "@/lib/documents/pdf-text-layer";
@@ -170,6 +169,12 @@ export async function loadOwnedDocument(
 export async function prepareVisionInput(
   document: LoadedDocument,
   pdfSupported: boolean,
+  /**
+   * The most pages a text-layer read takes. The default is the render cap, the
+   * pages a vision read would have seen; the index reads as many pages as the
+   * local text index does, so the index it writes is complete.
+   */
+  opts: { textLayerPages?: number } = {},
 ): Promise<VisionInput> {
   let buffer: Buffer;
   try {
@@ -213,7 +218,7 @@ export async function prepareVisionInput(
     }
     const read = await readPdfTextLayer(
       buffer,
-      RASTER_MAX_PAGES,
+      opts.textLayerPages ?? RASTER_MAX_PAGES,
       "documents.vision",
     );
     if (read.ok) {
@@ -221,7 +226,6 @@ export async function prepareVisionInput(
         ok: true,
         mode: "text",
         text: textLayerText(read.layer),
-        partial: isPartialTextLayer(read.layer),
       };
     }
     if (read.reason === "busy") return { ok: false, reason: "readerBusy" };

src/lib/documents/ai-runs/__tests__/unrenderable-pdf-runs.test.ts

@@ -363,6 +363,22 @@ describe("on a server that cannot render PDF pages", () => {
     );
   }, 30_000);
 
+  it("Read with AI indexes every page, not only the pages a vision read is capped to", async () => {
+    stored(pdfWith(12, 0));
+    const outcome = await executeDocumentIndex({
+      userId: "u1",
+      document,
+      input: { mode: "vision", pick: pick(recording([], "x")), budget },
+      origin,
+    });
+    expect(outcome).toMatchObject({ ok: true });
+    const text = (
+      vi.mocked(upsertContentIndex).mock.calls[0][0] as { text: string }
+    ).text;
+    // Twelve pages, each carrying the lab line once.
+    expect(text.split(LAB_LINE)).toHaveLength(13);
+  }, 30_000);
+
   it("Read with AI answers a scan with pdfRendererUnavailable and records why", async () => {
     stored(pdfWith(0, 1));
     const outcome = await executeDocumentIndex({

src/lib/documents/ai-runs/index-run.ts

@@ -29,6 +29,7 @@ import {
   transcribeDocument,
 } from "@/lib/documents/describe";
 import { recordIndexAttempt } from "@/lib/documents/index-document";
+import { LOCAL_MAX_PAGES } from "@/lib/documents/local-extract";
 import type { requireDocumentVisionProvider } from "@/lib/documents/provider-order";
 import { annotate } from "@/lib/logging/context";
 
@@ -154,7 +155,11 @@ export async function executeDocumentIndex(args: {
   const { pick, budget } = input;
   const servedBy = pick.entry.providerType as ProviderChainType;
 
-  const vision = await prepareVisionInput(document, pick.pdfSupported);
+  // A text-layer read here IS the index, so it takes every page the local
+  // text index would, not only the pages a vision read is capped to.
+  const vision = await prepareVisionInput(document, pick.pdfSupported, {
+    textLayerPages: LOCAL_MAX_PAGES,
+  });
   if (!vision.ok) {
     // Preparation failed before any provider dispatch — the reservation and
     // the slot go back. Refs #776 — each failure is also recorded on the row

src/lib/documents/local-extract.ts

@@ -46,7 +46,7 @@ import {
 export const LOCAL_TEXT_MIN_CHARS = 24;
 
 /** Cap on pages parsed — bounds CPU/time on a pathological many-page PDF. */
-const LOCAL_MAX_PAGES = 200;
+export const LOCAL_MAX_PAGES = 200;
 
 /** Provenance tag written to `DocumentContentIndex.source` on the local path. */
 export type LocalExtractSource = "local-pdf" | "local-ocr";

src/lib/documents/pdf-text-layer.ts

@@ -13,8 +13,7 @@
  *
  * A text read loses what only the picture carries (a table's ruling, a
  * stamp, a handwritten note), and the pages without a text layer are left
- * out. Callers that report a partial read report this one as partial
- * (`isPartialTextLayer`).
+ * out; the extraction and the lab scan report those pages.
  */
 import { annotate } from "@/lib/logging/context";
 
@@ -61,11 +60,6 @@ export function isUsablePageText(text: string): boolean {
   return text.replace(/\s+/g, "").length >= TEXT_LAYER_PAGE_MIN_CHARS;
 }
 
-/** Pages were left out: past the cap, or without a text layer. */
-export function isPartialTextLayer(layer: PdfTextLayer): boolean {
-  return layer.pagesWithoutText.length > 0 || layer.pageCount > layer.pagesRead;
-}
-
 /** The usable pages' text, one page after another. */
 export function textLayerText(layer: PdfTextLayer): string {
   return layer.pages.map((p) => p.text).join("\n\n");

src/lib/documents/vision-input.ts

@@ -19,9 +19,9 @@ export type VisionInput =
   /**
    * #1311 — a PDF this server cannot render, for a provider that cannot take
    * it whole, read by its text layer instead (`./pdf-text-layer.ts`).
-   * `partial` when pages were left out: scanned pages, or pages past the cap.
+   * The pages without a text layer (scanned pages) are not in `text`.
    */
-  | { ok: true; mode: "text"; text: string; partial: boolean }
+  | { ok: true; mode: "text"; text: string }
   | {
       ok: false;
       // Refs #776 — `rasterFailed` (this PDF would not render on a raster-

src/lib/jobs/__tests__/document-content-index-backfill.test.ts

@@ -432,6 +432,12 @@ describe("runContentIndexBackfillForUser", () => {
         providerType: null,
       }),
     );
+    // The index reads every page the local index would, not the render cap.
+    expect(readPdfTextLayer).toHaveBeenCalledWith(
+      expect.anything(),
+      200,
+      expect.any(String),
+    );
     // The reservation went back: nothing was sent.
     expect(vi.mocked(reconcileSpend).mock.calls.at(-1)![2]).toBe(0);
     expect(result).toMatchObject({ indexed: 1, skipped: 0, failed: 0 });

src/lib/jobs/document-content-index-backfill.ts

@@ -55,6 +55,7 @@ import {
   upsertContentIndex,
 } from "@/lib/documents/content-index";
 import { transcribeDocument } from "@/lib/documents/describe";
+import { LOCAL_MAX_PAGES } from "@/lib/documents/local-extract";
 import { withPdfWork } from "@/lib/documents/pdf-isolation";
 import { getGlobalBoss } from "@/lib/jobs/boss-instance";
 import { resolveDocumentVisionProvider } from "@/lib/documents/provider-order";
@@ -323,7 +324,7 @@ export async function runContentIndexBackfillForUser(
       }
       const vision = await withPdfWork(
         { owner: userId, lane: "background" },
-        () => prepareVisionInput(document, pick.pdfSupported),
+        () => prepareForIndex(document, pick.pdfSupported),
       );
       if (!vision.ok) {
         // Not vision-indexable after all (undecryptable, unreadable bytes,
@@ -394,6 +395,19 @@ export async function runContentIndexBackfillForUser(
   };
 }
 
+/**
+ * The stored original as index input. A text-layer read is the index itself,
+ * so it takes every page the local text index reads, not the render cap.
+ */
+function prepareForIndex(
+  document: NonNullable<Awaited<ReturnType<typeof loadOwnedDocument>>>,
+  pdfSupported: boolean,
+) {
+  return prepareVisionInput(document, pdfSupported, {
+    textLayerPages: LOCAL_MAX_PAGES,
+  });
+}
+
 /**
  * One document's provider read (or its text-layer read, #1311) and its index
  * row, with the reservation settled either way. A provider miss on one document must not abort the