fix(labs): bound each text-layer read of an unrenderable PDF

`planTextLayerReads` grouped whole pages into the reads the deadline
allows, with no bound on a read's size, while the text path refuses a part
past `OCR_TEXT_READ_MAX_CHARS`. A dense report in a request's three reads
went out as reads of 12 000 characters and more, which ends at the output
limit or loses rows.

`planTextLayerChunks` now cuts a page longer than one read at line
boundaries, keeps a page per read while the deadline allows that many, and
otherwise packs pages into reads of at most twice the surface's target
(8 000 characters for the lab scan, 5 000 for the document extraction).
What the reads cannot take is read from the start, and the pages left over
are reported with the pages without a text layer: `unreadPages` on the lab
scan, `extractionIncomplete` on a document.

Refs #1311

src/lib/documents/__tests__/unrenderable-pdf-text-layer.test.ts

@@ -17,7 +17,11 @@ import { decryptDocumentContent } from "@/lib/documents/store";
 
 import { prepareVisionInput } from "../ai-route-support";
 import { __resetNativeCanvasSupportForTests } from "../native-canvas-support";
-import { planVisionReads, SYNCHRONOUS_READ_SCHEDULE } from "../ocr-plan";
+import {
+  planTextLayerChunks,
+  planVisionReads,
+  SYNCHRONOUS_READ_SCHEDULE,
+} from "../ocr-plan";
 import { readPdfTextLayer } from "../pdf-text-layer";
 
 /** A 1×1 PNG: a page carrying only this is a scan with no text layer. */
@@ -173,7 +177,7 @@ describe("on a server that cannot render PDF pages", () => {
         ok: true,
         mode: "text",
         totalPages: 3,
-        pagesWithoutText: [2],
+        unreadPages: [2],
       });
       if (!plan.ok || plan.mode !== "text") return;
       expect(plan.chunks.map((c) => c.pages)).toEqual([[1], [3]]);
@@ -248,3 +252,51 @@ describe("on a server that renders PDF pages", () => {
     ).resolves.toEqual({ ok: false, reason: "renderFailed" });
   }, 60_000);
 });
+
+describe("planTextLayerChunks", () => {
+  const page = (n: number, lines: number) => ({
+    page: n,
+    text: Array.from(
+      { length: lines },
+      (_, i) => `Analyte ${n}-${i} 12.${i} mg/dl 10.0 - 20.0`,
+    ).join("\n"),
+  });
+
+  it("reads a page per read while the deadline allows it", () => {
+    const { chunks, unreadPages } = planTextLayerChunks(
+      [page(1, 3), page(2, 3)],
+      5,
+      4000,
+    );
+    expect(chunks.map((c) => c.pages)).toEqual([[1], [2]]);
+    expect(unreadPages).toEqual([]);
+  });
+
+  it("splits a page longer than one read at line boundaries", () => {
+    // ~200 lines of ~40 characters: about 8 000 characters on one page.
+    const { chunks } = planTextLayerChunks([page(1, 200)], 10, 4000);
+    expect(chunks.length).toBeGreaterThan(1);
+    expect(chunks.every((c) => c.pages[0] === 1)).toBe(true);
+    expect(chunks.every((c) => c.text!.length <= 4000)).toBe(true);
+    expect(chunks.map((c) => c.text).join("\n")).toBe(page(1, 200).text);
+  });
+
+  it("never sends a read past twice the target, and names the pages left over", () => {
+    const pages = Array.from({ length: 12 }, (_, i) => page(i + 1, 75));
+    const { chunks, unreadPages } = planTextLayerChunks(pages, 3, 4000);
+    expect(chunks).toHaveLength(3);
+    expect(chunks.every((c) => c.text!.length <= 8000)).toBe(true);
+    const read = new Set(chunks.flatMap((c) => c.pages));
+    expect(unreadPages.length).toBeGreaterThan(0);
+    expect(unreadPages).toContain(12);
+    expect(read.has(1)).toBe(true);
+  });
+
+  it("packs short pages into fewer reads when the deadline is tight", () => {
+    const pages = Array.from({ length: 6 }, (_, i) => page(i + 1, 3));
+    const { chunks, unreadPages } = planTextLayerChunks(pages, 2, 4000);
+    expect(chunks.length).toBeLessThanOrEqual(2);
+    expect(chunks.flatMap((c) => c.pages)).toEqual([1, 2, 3, 4, 5, 6]);
+    expect(unreadPages).toEqual([]);
+  });
+});

src/lib/documents/extract.ts

@@ -504,6 +504,7 @@ export async function planInboundReads(
     source.mime,
     source.pdfSupported,
     schedule,
+    INBOUND_TEXT_READ_CHARS,
   );
   if (!plan.ok) return plan;
   if (plan.mode === "text") {
@@ -515,8 +516,7 @@ export async function planInboundReads(
         totalPages: plan.totalPages,
         ocrText: plan.ocrText,
       },
-      partial:
-        plan.pageCoverage !== undefined || plan.pagesWithoutText.length > 0,
+      partial: plan.pageCoverage !== undefined || plan.unreadPages.length > 0,
     };
   }
   return {

src/lib/documents/ocr-plan.ts

@@ -178,8 +178,9 @@ export type OcrVisionPlan =
   /**
    * #1311 — a PDF this server cannot render, for a provider that cannot take
    * it whole, read by its text layer: one text part per page group, as the
-   * rendered pages would have been. Pages without a text layer (scanned
-   * pages) are not read and are listed in `pagesWithoutText`.
+   * rendered pages would have been, each read under the text bound. Pages
+   * without a text layer (scanned pages), and pages the reads could not take
+   * in full, are listed in `unreadPages`.
    */
   | {
       ok: true;
@@ -189,7 +190,7 @@ export type OcrVisionPlan =
       ocrText: string;
       totalPages: number;
       pageCoverage?: { read: number; total: number };
-      pagesWithoutText: number[];
+      unreadPages: number[];
     }
   | {
       ok: false;
@@ -219,6 +220,7 @@ function coverage(
 async function planTextLayerReads(
   bytes: Buffer,
   schedule: OcrSchedule,
+  textReadChars: number,
 ): Promise<OcrVisionPlan> {
   const read = await readPdfTextLayer(bytes, OCR_MAX_PAGES, "documents.ocr");
   if (!read.ok) {
@@ -233,14 +235,10 @@ async function planTextLayerReads(
     };
   }
   const { layer } = read;
-  const chunks = planPageReads(layer.pages.length, maxReadsFor(schedule)).map(
-    (group) => {
-      const pages = group.map((i) => layer.pages[i - 1]);
-      return {
-        text: pages.map((p) => p.text).join("\n\n"),
-        pages: pages.map((p) => p.page),
-      };
-    },
+  const { chunks, unreadPages } = planTextLayerChunks(
+    layer.pages,
+    maxReadsFor(schedule),
+    textReadChars,
   );
   return {
     ok: true,
@@ -249,7 +247,74 @@ async function planTextLayerReads(
     ocrText: textLayerText(layer),
     totalPages: layer.pageCount,
     ...coverage(layer.pagesRead, layer.pageCount),
-    pagesWithoutText: layer.pagesWithoutText,
+    unreadPages: [...new Set([...layer.pagesWithoutText, ...unreadPages])].sort(
+      (a, b) => a - b,
+    ),
+  };
+}
+
+/**
+ * The reads of a text layer, page-aware and under the same per-read bound as
+ * any other text: a page longer than `target` is read in parts at line
+ * boundaries, each part its own read while the deadline allows that many;
+ * otherwise the parts are packed into at most `maxReads` reads of about the
+ * share the deadline leaves, never past twice `target` (the most one read's
+ * answer is sized for). When the reads cannot take it all, the text is read
+ * from its start and the pages left over, wholly or in part, are returned in
+ * `unreadPages`, as `planTextReads` counts its `unreadParts`.
+ */
+export function planTextLayerChunks(
+  pages: { page: number; text: string }[],
+  maxReads: number,
+  target: number,
+): { chunks: OcrChunk[]; unreadPages: number[] } {
+  const reads = Math.max(1, Math.floor(maxReads));
+  const units = pages.flatMap(({ page, text }) =>
+    splitOcrText(text, Number.POSITIVE_INFINITY, target)
+      // A single line past the target stays whole in `splitOcrText`; one
+      // that long is cut, so no read can exceed the bound.
+      .flatMap((part) =>
+        part.length <= target
+          ? [part]
+          : Array.from({ length: Math.ceil(part.length / target) }, (_, i) =>
+              part.slice(i * target, (i + 1) * target),
+            ),
+      )
+      .map((part) => ({ page, text: part })),
+  );
+  // As many reads as parts: a page (or a page part) per read, as rendered
+  // pages are read.
+  if (units.length <= reads) {
+    return {
+      chunks: units.map((u) => ({ text: u.text, pages: [u.page] })),
+      unreadPages: [],
+    };
+  }
+  const total = units.reduce((n, u) => n + u.text.length, 0);
+  const size = Math.min(target * 2, Math.max(target, Math.ceil(total / reads)));
+  const packed: { page: number; text: string }[][] = [];
+  let current: { page: number; text: string }[] = [];
+  let length = 0;
+  for (const unit of units) {
+    if (current.length > 0 && length + 2 + unit.text.length > size) {
+      packed.push(current);
+      current = [];
+      length = 0;
+    }
+    current.push(unit);
+    length += (current.length > 1 ? 2 : 0) + unit.text.length;
+  }
+  if (current.length > 0) packed.push(current);
+  const kept = packed.slice(0, reads);
+  const unreadPages = [
+    ...new Set(packed.slice(reads).flatMap((c) => c.map((u) => u.page))),
+  ];
+  return {
+    chunks: kept.map((chunk) => ({
+      text: chunk.map((u) => u.text).join("\n\n"),
+      pages: [...new Set(chunk.map((u) => u.page))],
+    })),
+    unreadPages,
   };
 }
 
@@ -268,6 +333,8 @@ export async function planVisionReads(
   mime: OcrVisionMime,
   pdfSupported: boolean,
   schedule: OcrSchedule,
+  /** Text per read when the PDF is read by its text layer (the surface's own). */
+  textReadChars: number = OCR_TEXT_READ_CHARS,
 ): Promise<OcrVisionPlan> {
   if (mime !== "application/pdf") {
     return {
@@ -312,7 +379,7 @@ export async function planVisionReads(
     if (raster.reason !== "unsupported") {
       return { ok: false, reason: "renderFailed" };
     }
-    return planTextLayerReads(bytes, schedule);
+    return planTextLayerReads(bytes, schedule, textReadChars);
   }
   return {
     ok: true,

src/lib/labs/__tests__/ocr-run.test.ts

@@ -817,4 +817,37 @@ describe("a PDF on a server that cannot render PDF pages (#1311)", () => {
     expect(args.mode).toBe("vision");
     expect(args.chunks[0].documents).toHaveLength(1);
   }, 30_000);
+
+  it("keeps every text read of a dense report under the per-read bound", async () => {
+    // Twelve pages of about 3 000 characters each, and a request's three
+    // reads: grouping whole pages would send reads of 12 000 characters.
+    const doc = new jsPDF();
+    for (let page = 0; page < 12; page++) {
+      if (page > 0) doc.addPage();
+      for (let line = 0; line < 34; line++) {
+        doc.text(
+          `Analyte ${page}-${line} 12.${line} mg/dl 10.0 - 20.0 ref lab A`,
+          10,
+          8 + line * 8,
+        );
+      }
+    }
+    vi.mocked(runOcrExtraction).mockImplementation(extraction());
+    const out = await executeOcrExtraction({
+      userId: "u1",
+      input: { ...pdf(), bytes: Buffer.from(doc.output("arraybuffer")) },
+      budget,
+      schedule: schedule(6),
+    });
+    const args = vi.mocked(runOcrExtraction).mock.calls[0][0];
+    expect(args.chunks.length).toBeLessThanOrEqual(3);
+    for (const chunk of args.chunks) {
+      expect(chunk.text!.length).toBeLessThanOrEqual(OCR_TEXT_READ_MAX_CHARS);
+    }
+    // What the three reads could not take is named, not silently dropped.
+    expect(out.ok).toBe(true);
+    if (!out.ok) return;
+    expect(out.data.unreadPages?.length).toBeGreaterThan(0);
+    expect(out.data.unreadPages).toContain(12);
+  }, 30_000);
 });

src/lib/labs/ocr-run.ts

@@ -260,11 +260,13 @@ export async function executeOcrExtraction(args: {
   await schedule.arm(chunks.length * OCR_EXTRACT_MODEL_CALLS);
   const outcome = await read({ mode: plan.mode, chunks, totalPages });
   if (!outcome.ok) return outcome;
-  // #1311 — a text-layer read leaves out the pages without a text layer;
-  // they are named as pages the scan did not read.
+  // #1311 — a text-layer read leaves out the pages without a text layer, and
+  // any the reads could not take in full; they are named as pages not read.
   const unreadPages = [
-    ...(outcome.data.unreadPages ?? []),
-    ...(plan.mode === "text" ? plan.pagesWithoutText : []),
+    ...new Set([
+      ...(outcome.data.unreadPages ?? []),
+      ...(plan.mode === "text" ? plan.unreadPages : []),
+    ]),
   ].sort((a, b) => a - b);
   return {
     ok: true,