From 253176d01aa282161a15030d35b0b0770bed64c3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Tue, 3 Nov 2026 23:24:49 +0100 Subject: [PATCH] fix(documents): index every page of a PDF read by its text layer "Read with AI" and the index backfill wrote a text-layer read as a complete `local-pdf` index although it held only the first ten pages (the render cap), while the local text index reads up to 200. `prepareVisionInput` takes the number of pages a text-layer read may take, and both index writers ask for the local index's own cap, so the index is complete. The `partial` flag on a text read had no reader and is gone; the extraction and the lab scan report their missing pages themselves. Refs #1311 --- .../__tests__/index-document.test.ts | 1 - .../unrenderable-pdf-text-layer.test.ts | 21 ++++++++++++------- src/lib/documents/ai-route-support.ts | 10 ++++++--- .../__tests__/unrenderable-pdf-runs.test.ts | 16 ++++++++++++++ src/lib/documents/ai-runs/index-run.ts | 7 ++++++- src/lib/documents/local-extract.ts | 2 +- src/lib/documents/pdf-text-layer.ts | 8 +------ src/lib/documents/vision-input.ts | 4 ++-- .../document-content-index-backfill.test.ts | 6 ++++++ .../jobs/document-content-index-backfill.ts | 16 +++++++++++++- 10 files changed, 68 insertions(+), 23 deletions(-) diff --git a/src/lib/documents/__tests__/index-document.test.ts b/src/lib/documents/__tests__/index-document.test.ts index baa5b0a4e..dba8b7458 100644 --- a/src/lib/documents/__tests__/index-document.test.ts +++ b/src/lib/documents/__tests__/index-document.test.ts @@ -255,7 +255,6 @@ describe("indexDocumentContent — local fallback path", () => { ok: true, mode: "text", text: "Haemoglobin 14.2 g/dl", - partial: false, }); const outcome = await indexDocumentContent("user-1", "doc-1"); diff --git a/src/lib/documents/__tests__/unrenderable-pdf-text-layer.test.ts b/src/lib/documents/__tests__/unrenderable-pdf-text-layer.test.ts index 8984309dd..9ec82dc7a 100644 --- a/src/lib/documents/__tests__/unrenderable-pdf-text-layer.test.ts +++ b/src/lib/documents/__tests__/unrenderable-pdf-text-layer.test.ts @@ -125,19 +125,26 @@ describe("on a server that cannot render PDF pages", () => { it("reads a text-layer PDF as text for a provider that only takes images", async () => { stored(buildPdf([LAB_PAGE, LETTER_PAGE])); const vision = await prepareVisionInput(DOC, false); - expect(vision).toMatchObject({ ok: true, mode: "text", partial: false }); + expect(vision).toMatchObject({ ok: true, mode: "text" }); if (!vision.ok || vision.mode !== "text") return; expect(vision.text).toContain("Creatinine 0.9 mg/dl"); expect(vision.text).toContain("community-acquired pneumonia"); }, 30_000); - it("marks the text read partial when a page is a scan", async () => { - stored(buildPdf([LAB_PAGE, SCAN])); - await expect(prepareVisionInput(DOC, false)).resolves.toMatchObject({ - ok: true, - mode: "text", - partial: true, + it("reads the render cap's pages by default, and as many as asked for the index", async () => { + const pages = Array.from({ length: 12 }, (_, i) => ({ + text: `Page ${i + 1} creatinine 0.9 mg/dl reference 0.7 - 1.2`, + })); + stored(buildPdf(pages)); + const capped = await prepareVisionInput(DOC, false); + const whole = await prepareVisionInput(DOC, false, { + textLayerPages: 200, }); + if (!capped.ok || capped.mode !== "text") throw new Error("no text"); + if (!whole.ok || whole.mode !== "text") throw new Error("no text"); + expect(capped.text).toContain("Page 10 "); + expect(capped.text).not.toContain("Page 11 "); + expect(whole.text).toContain("Page 12 "); }, 30_000); it("answers a scan with pdfRendererUnavailable, never as a read", async () => { diff --git a/src/lib/documents/ai-route-support.ts b/src/lib/documents/ai-route-support.ts index 153547190..ee21b8edc 100644 --- a/src/lib/documents/ai-route-support.ts +++ b/src/lib/documents/ai-route-support.ts @@ -13,7 +13,6 @@ import { Buffer } from "node:buffer"; import { apiError } from "@/lib/api-response"; import { prisma } from "@/lib/db"; import { - isPartialTextLayer, readPdfTextLayer, textLayerText, } from "@/lib/documents/pdf-text-layer"; @@ -170,6 +169,12 @@ export async function loadOwnedDocument( export async function prepareVisionInput( document: LoadedDocument, pdfSupported: boolean, + /** + * The most pages a text-layer read takes. The default is the render cap, the + * pages a vision read would have seen; the index reads as many pages as the + * local text index does, so the index it writes is complete. + */ + opts: { textLayerPages?: number } = {}, ): Promise { let buffer: Buffer; try { @@ -213,7 +218,7 @@ export async function prepareVisionInput( } const read = await readPdfTextLayer( buffer, - RASTER_MAX_PAGES, + opts.textLayerPages ?? RASTER_MAX_PAGES, "documents.vision", ); if (read.ok) { @@ -221,7 +226,6 @@ export async function prepareVisionInput( ok: true, mode: "text", text: textLayerText(read.layer), - partial: isPartialTextLayer(read.layer), }; } if (read.reason === "busy") return { ok: false, reason: "readerBusy" }; diff --git a/src/lib/documents/ai-runs/__tests__/unrenderable-pdf-runs.test.ts b/src/lib/documents/ai-runs/__tests__/unrenderable-pdf-runs.test.ts index 8bd2295d8..10e27bf10 100644 --- a/src/lib/documents/ai-runs/__tests__/unrenderable-pdf-runs.test.ts +++ b/src/lib/documents/ai-runs/__tests__/unrenderable-pdf-runs.test.ts @@ -363,6 +363,22 @@ describe("on a server that cannot render PDF pages", () => { ); }, 30_000); + it("Read with AI indexes every page, not only the pages a vision read is capped to", async () => { + stored(pdfWith(12, 0)); + const outcome = await executeDocumentIndex({ + userId: "u1", + document, + input: { mode: "vision", pick: pick(recording([], "x")), budget }, + origin, + }); + expect(outcome).toMatchObject({ ok: true }); + const text = ( + vi.mocked(upsertContentIndex).mock.calls[0][0] as { text: string } + ).text; + // Twelve pages, each carrying the lab line once. + expect(text.split(LAB_LINE)).toHaveLength(13); + }, 30_000); + it("Read with AI answers a scan with pdfRendererUnavailable and records why", async () => { stored(pdfWith(0, 1)); const outcome = await executeDocumentIndex({ diff --git a/src/lib/documents/ai-runs/index-run.ts b/src/lib/documents/ai-runs/index-run.ts index ec05fa142..3b1109cb1 100644 --- a/src/lib/documents/ai-runs/index-run.ts +++ b/src/lib/documents/ai-runs/index-run.ts @@ -29,6 +29,7 @@ import { transcribeDocument, } from "@/lib/documents/describe"; import { recordIndexAttempt } from "@/lib/documents/index-document"; +import { LOCAL_MAX_PAGES } from "@/lib/documents/local-extract"; import type { requireDocumentVisionProvider } from "@/lib/documents/provider-order"; import { annotate } from "@/lib/logging/context"; @@ -154,7 +155,11 @@ export async function executeDocumentIndex(args: { const { pick, budget } = input; const servedBy = pick.entry.providerType as ProviderChainType; - const vision = await prepareVisionInput(document, pick.pdfSupported); + // A text-layer read here IS the index, so it takes every page the local + // text index would, not only the pages a vision read is capped to. + const vision = await prepareVisionInput(document, pick.pdfSupported, { + textLayerPages: LOCAL_MAX_PAGES, + }); if (!vision.ok) { // Preparation failed before any provider dispatch — the reservation and // the slot go back. Refs #776 — each failure is also recorded on the row diff --git a/src/lib/documents/local-extract.ts b/src/lib/documents/local-extract.ts index 9507e89a9..9df76545b 100644 --- a/src/lib/documents/local-extract.ts +++ b/src/lib/documents/local-extract.ts @@ -46,7 +46,7 @@ import { export const LOCAL_TEXT_MIN_CHARS = 24; /** Cap on pages parsed — bounds CPU/time on a pathological many-page PDF. */ -const LOCAL_MAX_PAGES = 200; +export const LOCAL_MAX_PAGES = 200; /** Provenance tag written to `DocumentContentIndex.source` on the local path. */ export type LocalExtractSource = "local-pdf" | "local-ocr"; diff --git a/src/lib/documents/pdf-text-layer.ts b/src/lib/documents/pdf-text-layer.ts index 7652ab292..b61a97c8b 100644 --- a/src/lib/documents/pdf-text-layer.ts +++ b/src/lib/documents/pdf-text-layer.ts @@ -13,8 +13,7 @@ * * A text read loses what only the picture carries (a table's ruling, a * stamp, a handwritten note), and the pages without a text layer are left - * out. Callers that report a partial read report this one as partial - * (`isPartialTextLayer`). + * out; the extraction and the lab scan report those pages. */ import { annotate } from "@/lib/logging/context"; @@ -61,11 +60,6 @@ export function isUsablePageText(text: string): boolean { return text.replace(/\s+/g, "").length >= TEXT_LAYER_PAGE_MIN_CHARS; } -/** Pages were left out: past the cap, or without a text layer. */ -export function isPartialTextLayer(layer: PdfTextLayer): boolean { - return layer.pagesWithoutText.length > 0 || layer.pageCount > layer.pagesRead; -} - /** The usable pages' text, one page after another. */ export function textLayerText(layer: PdfTextLayer): string { return layer.pages.map((p) => p.text).join("\n\n"); diff --git a/src/lib/documents/vision-input.ts b/src/lib/documents/vision-input.ts index bd7c1b0f2..6bbe5bda3 100644 --- a/src/lib/documents/vision-input.ts +++ b/src/lib/documents/vision-input.ts @@ -19,9 +19,9 @@ export type VisionInput = /** * #1311 — a PDF this server cannot render, for a provider that cannot take * it whole, read by its text layer instead (`./pdf-text-layer.ts`). - * `partial` when pages were left out: scanned pages, or pages past the cap. + * The pages without a text layer (scanned pages) are not in `text`. */ - | { ok: true; mode: "text"; text: string; partial: boolean } + | { ok: true; mode: "text"; text: string } | { ok: false; // Refs #776 — `rasterFailed` (this PDF would not render on a raster- diff --git a/src/lib/jobs/__tests__/document-content-index-backfill.test.ts b/src/lib/jobs/__tests__/document-content-index-backfill.test.ts index ef91897f3..243a2de10 100644 --- a/src/lib/jobs/__tests__/document-content-index-backfill.test.ts +++ b/src/lib/jobs/__tests__/document-content-index-backfill.test.ts @@ -432,6 +432,12 @@ describe("runContentIndexBackfillForUser", () => { providerType: null, }), ); + // The index reads every page the local index would, not the render cap. + expect(readPdfTextLayer).toHaveBeenCalledWith( + expect.anything(), + 200, + expect.any(String), + ); // The reservation went back: nothing was sent. expect(vi.mocked(reconcileSpend).mock.calls.at(-1)![2]).toBe(0); expect(result).toMatchObject({ indexed: 1, skipped: 0, failed: 0 }); diff --git a/src/lib/jobs/document-content-index-backfill.ts b/src/lib/jobs/document-content-index-backfill.ts index d6e181a4e..dc88c595c 100644 --- a/src/lib/jobs/document-content-index-backfill.ts +++ b/src/lib/jobs/document-content-index-backfill.ts @@ -55,6 +55,7 @@ import { upsertContentIndex, } from "@/lib/documents/content-index"; import { transcribeDocument } from "@/lib/documents/describe"; +import { LOCAL_MAX_PAGES } from "@/lib/documents/local-extract"; import { withPdfWork } from "@/lib/documents/pdf-isolation"; import { getGlobalBoss } from "@/lib/jobs/boss-instance"; import { resolveDocumentVisionProvider } from "@/lib/documents/provider-order"; @@ -323,7 +324,7 @@ export async function runContentIndexBackfillForUser( } const vision = await withPdfWork( { owner: userId, lane: "background" }, - () => prepareVisionInput(document, pick.pdfSupported), + () => prepareForIndex(document, pick.pdfSupported), ); if (!vision.ok) { // Not vision-indexable after all (undecryptable, unreadable bytes, @@ -394,6 +395,19 @@ export async function runContentIndexBackfillForUser( }; } +/** + * The stored original as index input. A text-layer read is the index itself, + * so it takes every page the local text index reads, not the render cap. + */ +function prepareForIndex( + document: NonNullable>>, + pdfSupported: boolean, +) { + return prepareVisionInput(document, pdfSupported, { + textLayerPages: LOCAL_MAX_PAGES, + }); +} + /** * One document's provider read (or its text-layer read, #1311) and its index * row, with the reservation settled either way. A provider miss on one document must not abort the