/** * Local (provider-free) text extraction for the document vault. * * When NO AI provider is configured — or the user has not consented to one — * content search still needs a baseline. This module extracts a PDF's embedded * text layer SERVER-SIDE with `pdf-parse` (already a direct dependency; its * `getText()` touches only the pure-JS/wasm pdfjs core, never the native * `@napi-rs/canvas`), in milliseconds, with no OCR and no third-party egress. * The extracted text feeds the SAME blind, encrypted content index the provider * path writes (`upsertContentIndex`), so nothing readable is stored at rest. * * Digitally-generated PDFs (lab reports, discharge letters, referrals — the * clinical common case) carry a real embedded text layer this path reads. * Scanned/image-only PDFs and uploaded images have NO text layer: local OCR * (server-side tesseract + `deu+eng` traineddata baked into the image, plus * `@napi-rs/canvas` rasterisation for scanned PDFs) is a deferred follow-up, NOT * built here — it carries the only real Alpine/standalone-tracing + CPU cost. * Until it lands, those documents stay un-indexed on the local path and are * covered by the provider (vision) path whenever one is configured. */ import { Buffer } from "node:buffer"; // `pdf-parse` pulls in pdfjs, whose module top-level references the browser-only // `DOMMatrix` global. A STATIC import makes the Turbopack server chunk evaluate // that reference at instantiation and throw `DOMMatrix is not defined`, taking // down every route/worker that shares the chunk (the document routes among // them). The value is therefore loaded lazily inside `extractPdfText` via a // runtime `import()`, which lets pdf-parse initialise pdfjs on its own terms and // never touches DOMMatrix at load — verified to extract a real text layer under // plain Node. Keep the surface type as a type-only import (fully erased at // build, no runtime module evaluation). import type { PDFParse } from "pdf-parse"; import { annotate } from "@/lib/logging/context"; /** * Minimum joined text length for a PDF to count as a real text-layer document. * Below this a PDF is treated as scanned/empty — a few stray glyphs from a form * field or a page number are not a usable index. */ export const LOCAL_TEXT_MIN_CHARS = 24; /** Cap on pages parsed — bounds CPU/time on a pathological many-page PDF. */ const LOCAL_MAX_PAGES = 200; /** Provenance tag written to `DocumentContentIndex.source` on the local path. */ export type LocalExtractSource = "local-pdf" | "local-ocr"; /** * Outcome of a local extraction attempt. * - `ok` → usable text was recovered (index it). * - `empty` → a valid document with no usable text layer (scanned PDF); * a provider vision pass is the only way to read it. * - `unsupported` → a MIME the local path cannot read without OCR (images) or * at all; leave un-indexed locally. * - `error` → the file was malformed / unreadable; treat as un-indexed. */ export type LocalExtractResult = | { ok: true; text: string; source: LocalExtractSource } | { ok: false; reason: "empty" | "unsupported" | "error" }; /** * Extract a text-layer PDF's embedded text server-side. Returns `empty` for a * scanned/no-text-layer PDF and `error` for a malformed/encrypted one — never * throws, so a bad document can never abort the caller (the upload/backfill job * just leaves it un-indexed). */ export async function extractPdfText( buffer: Buffer, ): Promise { let parser: PDFParse | null = null; try { // Lazy load: keeps pdfjs/DOMMatrix out of the eager server chunk (see the // import-site note). Resolved at first PDF extraction, then module-cached. const { PDFParse } = await import("pdf-parse"); parser = new PDFParse({ data: new Uint8Array(buffer) }); const result = await parser.getText({ // Bound the page walk and drop the default "-- N of M --" page markers so // only real document text is tokenised. first: LOCAL_MAX_PAGES, pageJoiner: "", }); const text = (result.text ?? "").trim(); if (text.length < LOCAL_TEXT_MIN_CHARS) { return { ok: false, reason: "empty" }; } return { ok: true, text, source: "local-pdf" }; } catch { return { ok: false, reason: "error" }; } finally { if (parser) { try { await parser.destroy(); } catch { // Best-effort teardown; a destroy failure must not mask the result. } } } } /** * Dispatch local extraction by MIME. PDFs go through the text-layer reader; * images are the deferred server-OCR seam (see the module header) and return * `unsupported` so the caller falls through to the provider path or leaves the * document un-indexed. */ export async function localExtractText( buffer: Buffer, mime: string, ): Promise { if (mime === "application/pdf") { return extractPdfText(buffer); } if (mime.startsWith("image/")) { // SEAM (NOT DONE): server-side tesseract OCR for scanned images/PDFs is a // deferred follow-up. It needs the `deu+eng` traineddata baked into the // image and (for scanned PDFs) `@napi-rs/canvas` traced into the standalone // build — the only real deployment cost in this area. Until it ships, a // no-provider image stays un-indexed locally; a configured vision provider // covers it on the AI-first path. annotate({ action: { name: "documents.localIndex.ocrNotImplemented" }, meta: { mime }, }); return { ok: false, reason: "unsupported" }; } return { ok: false, reason: "unsupported" }; }