diff --git a/src/lib/documents/pdf-child.mjs b/src/lib/documents/pdf-child.mjs index 1bb7c7665..6d460f60d 100644 --- a/src/lib/documents/pdf-child.mjs +++ b/src/lib/documents/pdf-child.mjs @@ -15,7 +15,7 @@ import { writeFileSync } from "node:fs"; import * as nodeModule from "node:module"; import { renderImageThumbnail } from "./image-engine.mjs"; -import { readTextLayer, renderPdfPages } from "./pdf-engine.mjs"; +import { readPageTexts, readTextLayer, renderPdfPages } from "./pdf-engine.mjs"; // The parent closed the canvas gate (`./native-canvas-support.ts`): this CPU // lacks instructions the native canvas was built with, and loading it ends @@ -43,6 +43,7 @@ try { /** * @typedef {{ op: "text"; data: Uint8Array; maxPages: number } + * | { op: "page-texts"; data: Uint8Array; maxPages: number } * | { op: "render"; data: Uint8Array; options: import("./pdf-engine.mjs").RenderOptions } * | { op: "image-thumbnail"; data: Uint8Array; options: import("./image-engine.mjs").ImageThumbnailOptions }} PdfChildRequest */ @@ -68,6 +69,12 @@ async function handle(request) { if (request.op === "text") { return { ok: true, text: await readTextLayer(data, request.maxPages) }; } + if (request.op === "page-texts") { + return { + ok: true, + pageTexts: await readPageTexts(data, request.maxPages), + }; + } if (request.op === "render") { return { ok: true, diff --git a/src/lib/documents/pdf-engine.mjs b/src/lib/documents/pdf-engine.mjs index a5e001325..3b67330fe 100644 --- a/src/lib/documents/pdf-engine.mjs +++ b/src/lib/documents/pdf-engine.mjs @@ -270,6 +270,42 @@ export async function readTextLayer(data, maxPages = Number.POSITIVE_INFINITY) { } } +/** + * The text layer page by page, without rendering: the first `maxPages` pages, + * each read the way `pageText` reads a rendered page (line by line, cut and + * marked past `RASTER_PAGE_TEXT_MAX`, empty for a page without one), and the + * document's page count. Throws what pdf.js throws when the document will not + * open; a page that will not parse reads as empty. + * + * @param {Uint8Array} data + * @param {number} maxPages + * @returns {Promise<{ pageCount: number; texts: string[] }>} + */ +export async function readPageTexts(data, maxPages) { + const task = await openPdfDocument(data); + try { + const doc = await task.promise; + const count = Math.min(doc.numPages, maxPages); + /** @type {string[]} */ + const texts = []; + for (let n = 1; n <= count; n++) { + const page = await doc.getPage(n); + try { + texts.push(await pageText(page)); + } finally { + page.cleanup(); + } + } + return { pageCount: doc.numPages, texts }; + } finally { + try { + await task.destroy(); + } catch { + // Best-effort teardown; a destroy failure must not mask the result. + } + } +} + /** * The most text kept from one page's text layer when pages are rendered. * Collection stops here while the items are read, so a PDF whose content diff --git a/src/lib/documents/pdf-isolation.ts b/src/lib/documents/pdf-isolation.ts index fa405ebd7..6937afa75 100644 --- a/src/lib/documents/pdf-isolation.ts +++ b/src/lib/documents/pdf-isolation.ts @@ -79,14 +79,23 @@ const CHILD_PATH = ["src", "lib", "documents", "pdf-child.mjs"]; export type PdfChildRequest = | { op: "text"; data: Uint8Array; maxPages: number } + | { op: "page-texts"; data: Uint8Array; maxPages: number } | { op: "render"; data: Uint8Array; options: RenderOptions } | { op: "image-thumbnail"; data: Uint8Array; options: ImageThumbnailOptions }; type PdfChildValue = R extends { op: "text" } ? string - : R extends { op: "render" } - ? RenderedDocument - : ImageThumbnailOutcome; + : R extends { op: "page-texts" } + ? PdfPageTexts + : R extends { op: "render" } + ? RenderedDocument + : ImageThumbnailOutcome; + +/** A document's text layer page by page: the first pages read, and how many it has. */ +export interface PdfPageTexts { + pageCount: number; + texts: string[]; +} /** Why a child gave no result. */ export type PdfChildFailure = @@ -212,6 +221,7 @@ async function residentMb(pid: number): Promise { interface ChildReply { ok: boolean; text?: string; + pageTexts?: PdfPageTexts; document?: RenderedDocument; thumbnail?: ImageThumbnailOutcome; error?: { name: string; message: string }; @@ -237,6 +247,9 @@ function decodeValue( reply: ChildReply, ): PdfChildValue { if (request.op === "text") return reply.text as PdfChildValue; + if (request.op === "page-texts") { + return reply.pageTexts as PdfChildValue; + } if (request.op === "render") { return decodeDocument( reply.document as RenderedDocument, diff --git a/src/lib/documents/pdf-text-layer.ts b/src/lib/documents/pdf-text-layer.ts new file mode 100644 index 000000000..7652ab292 --- /dev/null +++ b/src/lib/documents/pdf-text-layer.ts @@ -0,0 +1,127 @@ +/** + * Reading a PDF by its text layer when this server cannot render its pages + * (#1311). + * + * The page renderer needs CPU features some hosts lack + * (`./native-canvas-support.ts`). There a provider that takes a PDF whole + * still reads it, but a provider that only takes images gets nothing to look + * at. A digitally produced report states every word and digit in its text + * layer, and pdf.js reads that without rendering, so such a PDF is read as + * text instead. A scanned page has no text layer, and this never pretends it + * does: a page counts only with `TEXT_LAYER_PAGE_MIN_CHARS` characters of its + * own, and a PDF with no such page is not read at all. + * + * A text read loses what only the picture carries (a table's ruling, a + * stamp, a handwritten note), and the pages without a text layer are left + * out. Callers that report a partial read report this one as partial + * (`isPartialTextLayer`). + */ +import { annotate } from "@/lib/logging/context"; + +import { + PdfReaderUnavailableError, + PdfResourceLimitError, + readPdfPageTexts, +} from "./pdfjs"; + +/** + * Characters other than white space a page's text layer needs before the page + * counts as read. A scanner that stamps a page number, a date or a fax header + * onto an image leaves a few; a page of a report, even a short letter, has + * hundreds. + */ +export const TEXT_LAYER_PAGE_MIN_CHARS = 40; + +/** A PDF's usable text layer, page by page. */ +export interface PdfTextLayer { + /** Pages in the PDF. Above the pages looked at when the cap cut it short. */ + pageCount: number; + /** Pages looked at: the first `pagesRead` of `pageCount`. */ + pagesRead: number; + /** The pages with a usable text layer (1-based), in order. */ + pages: { page: number; text: string }[]; + /** Pages looked at that have none (a scanned page): not read. */ + pagesWithoutText: number[]; +} + +export type PdfTextLayerRead = + | { ok: true; layer: PdfTextLayer } + | { + ok: false; + /** + * `none`: no page has a usable text layer (a scan). `unreadable`: + * pdf.js could not open the file, or it crossed the reader's bounds. + * `busy`: the reader could not take it now; not a verdict on the file. + */ + reason: "none" | "unreadable" | "busy"; + }; + +/** Whether one page's text layer is enough to count the page as read. */ +export function isUsablePageText(text: string): boolean { + return text.replace(/\s+/g, "").length >= TEXT_LAYER_PAGE_MIN_CHARS; +} + +/** Pages were left out: past the cap, or without a text layer. */ +export function isPartialTextLayer(layer: PdfTextLayer): boolean { + return layer.pagesWithoutText.length > 0 || layer.pageCount > layer.pagesRead; +} + +/** The usable pages' text, one page after another. */ +export function textLayerText(layer: PdfTextLayer): string { + return layer.pages.map((p) => p.text).join("\n\n"); +} + +/** + * The usable text layer of the first `maxPages` pages. Never throws; `surface` + * names the caller on the wide event. + */ +export async function readPdfTextLayer( + bytes: Uint8Array, + maxPages: number, + surface: string, +): Promise { + let read: { pageCount: number; texts: string[] }; + try { + read = await readPdfPageTexts(bytes, maxPages); + } catch (err) { + const busy = err instanceof PdfReaderUnavailableError; + annotate({ + action: { name: "documents.pdfTextLayer.failed" }, + meta: { + surface, + reason: busy + ? err.kind + : err instanceof PdfResourceLimitError + ? "resource_limit" + : "unreadable", + }, + }); + return { ok: false, reason: busy ? "busy" : "unreadable" }; + } + const pages: PdfTextLayer["pages"] = []; + const pagesWithoutText: number[] = []; + read.texts.forEach((raw, i) => { + const text = raw.trim(); + if (isUsablePageText(text)) pages.push({ page: i + 1, text }); + else pagesWithoutText.push(i + 1); + }); + annotate({ + action: { name: "documents.pdfTextLayer.read" }, + meta: { + surface, + pageCount: read.pageCount, + pagesRead: read.texts.length, + pagesWithText: pages.length, + }, + }); + if (pages.length === 0) return { ok: false, reason: "none" }; + return { + ok: true, + layer: { + pageCount: read.pageCount, + pagesRead: read.texts.length, + pages, + pagesWithoutText, + }, + }; +} diff --git a/src/lib/documents/pdfjs.ts b/src/lib/documents/pdfjs.ts index 7a55c75ef..988286a6b 100644 --- a/src/lib/documents/pdfjs.ts +++ b/src/lib/documents/pdfjs.ts @@ -7,7 +7,7 @@ * This module has no logging of its own, so the end-to-end suite can read * report text with it; the callers say what failed. */ -import { runPdfChild } from "./pdf-isolation"; +import { runPdfChild, type PdfPageTexts } from "./pdf-isolation"; export { joinTextRuns } from "./pdf-engine.mjs"; @@ -67,3 +67,28 @@ export async function readPdfText( err.name = result.name; throw err; } + +/** + * The text layer of the first `maxPages` pages, page by page, with the + * document's page count. Each page is read as a rendered page's text layer is + * (`pageText`: cut and marked past `RASTER_PAGE_TEXT_MAX`, empty for a page + * without one), and nothing is rendered, so it works where the page renderer + * cannot run. Throws as `readPdfText` does. + */ +export async function readPdfPageTexts( + data: Uint8Array, + maxPages: number, +): Promise { + const result = await runPdfChild( + { op: "page-texts", data, maxPages }, + { timeoutMs: PDF_TEXT_TIMEOUT_MS }, + ); + if (result.ok) return result.value; + if (result.reason === "limit") throw new PdfResourceLimitError(result.limit); + if (result.reason === "busy" || result.reason === "interrupted") { + throw new PdfReaderUnavailableError(result.reason); + } + const err = new Error(result.message); + err.name = result.name; + throw err; +}