diff --git a/next.config.ts b/next.config.ts index 92a373f73..63a17d195 100644 --- a/next.config.ts +++ b/next.config.ts @@ -242,7 +242,19 @@ const nextConfig: NextConfig = { // files explicitly keeps them in the runtime image alongside the // bundled module. outputFileTracingIncludes: { - "*": ["./src/lib/ai/prompts/safety-contracts.*.yaml"], + "*": [ + "./src/lib/ai/prompts/safety-contracts.*.yaml", + // Document AI local text extraction runs `pdf-parse` (→ pdfjs-dist) + // server-side in the content-index job — the provider-free fallback that + // keeps content search working with no AI configured. Both are pure JS + + // wasm; there is NO native `.node` on the text-layer path (`@napi-rs/canvas` + // is only pulled for image rendering, which `getText()` never invokes). + // The standalone tracer can miss pdfjs-dist's wasm decoders (they resolve + // via `new URL(..., import.meta.url)`), so pin both packages into the + // runtime image explicitly. pnpm keeps pdfjs-dist unhoisted under `.pnpm`. + "./node_modules/.pnpm/pdf-parse@*/node_modules/pdf-parse/**", + "./node_modules/.pnpm/pdfjs-dist@*/node_modules/pdfjs-dist/**", + ], }, // v1.4.34 IW-A — silence the Turbopack NFT trace warnings emitted // during `next build`. The tracer follows the `MAXMIND_LICENSE_KEY` diff --git a/prisma/schema.prisma b/prisma/schema.prisma index f3f5d2fa1..77dfe0df4 100644 --- a/prisma/schema.prisma +++ b/prisma/schema.prisma @@ -5965,7 +5965,10 @@ model DocumentContentIndex { /// (raw SQL in the migration; Prisma cannot express a GIN index). searchTokens String[] @default([]) @map("search_tokens") /// Provenance of the indexed text: "vision" (provider transcription of the - /// stored original) or "text-ocr" (browser-OCR text posted by the client). + /// stored original), "text-ocr" (browser-OCR text posted by the client), + /// "local-pdf" (server-side text-layer extraction, no provider) or "local-ocr" + /// (server-side OCR of a scanned document — deferred follow-up). Free String; + /// adding a source value needs no migration. source String /// Provider type that produced the text (diagnostic only), or null on the /// text-ocr path. diff --git a/src/app/api/documents/inbound/__tests__/route.test.ts b/src/app/api/documents/inbound/__tests__/route.test.ts index af2620cb2..7910c9ba4 100644 --- a/src/app/api/documents/inbound/__tests__/route.test.ts +++ b/src/app/api/documents/inbound/__tests__/route.test.ts @@ -62,6 +62,10 @@ vi.mock("@/lib/modules/gate", () => ({ requireModuleEnabled: vi.fn().mockResolvedValue({ enabled: true }), })); +vi.mock("@/lib/jobs/document-index", () => ({ + enqueueDocumentIndex: vi.fn().mockResolvedValue({ enqueued: true }), +})); + vi.mock("@/lib/rate-limit", () => ({ checkRateLimit: vi.fn().mockResolvedValue({ allowed: true, @@ -94,6 +98,7 @@ import { POST, GET } from "../route"; import { prisma } from "@/lib/db"; import { getSession } from "@/lib/auth/session"; import { requireModuleEnabled } from "@/lib/modules/gate"; +import { enqueueDocumentIndex } from "@/lib/jobs/document-index"; const post = POST as unknown as (r: Request) => Promise; const get = GET as unknown as (r: Request) => Promise; @@ -203,6 +208,9 @@ describe("POST /api/documents/inbound (vault upload)", () => { expect(stored.equals(PNG_BYTES)).toBe(false); // The create never round-trips the blob back out. expect(arg.omit).toEqual({ contentEncrypted: true }); + + // A fresh store auto-enqueues the per-document index job (owner + id). + expect(enqueueDocumentIndex).toHaveBeenCalledWith("user-1", "doc-1"); }); it("rejects an unidentifiable payload with 415 + reason unsupportedType", async () => { @@ -264,6 +272,8 @@ describe("POST /api/documents/inbound (vault upload)", () => { expect(where.contentSha256).toBe(PNG_SHA256); expect(where.deletedAt).toBeNull(); expect(txCreate).not.toHaveBeenCalled(); + // A duplicate short-circuits before the store — no index job is enqueued. + expect(enqueueDocumentIndex).not.toHaveBeenCalled(); }); it("pre-links the caller's episodes and refuses a foreign episode id", async () => { diff --git a/src/app/api/documents/inbound/route.ts b/src/app/api/documents/inbound/route.ts index b41273e4a..bf25857da 100644 --- a/src/app/api/documents/inbound/route.ts +++ b/src/app/api/documents/inbound/route.ts @@ -41,6 +41,7 @@ import { serialiseDocument, type SerialisableDocument, } from "@/lib/documents/store"; +import { enqueueDocumentIndex } from "@/lib/jobs/document-index"; import { detectDocumentType, resolveDocumentLimits, @@ -361,6 +362,13 @@ async function postUpload(request: Request): Promise { }, }); + // Auto-index the freshly stored document for content search: enqueue a + // fire-and-forget background job (provider-first, local text-layer fallback). + // The upload response never blocks on or fails because of indexing; a missing + // boss (worker not up) is a silent no-op. Only fresh inserts enqueue — a + // duplicate upload returns early above and never reaches here. + await enqueueDocumentIndex(user.id, document.id); + const links = await loadConditionLinks(user.id, [document.id]); return apiSuccess( serialiseDocument( diff --git a/src/lib/documents/__tests__/index-document.test.ts b/src/lib/documents/__tests__/index-document.test.ts new file mode 100644 index 000000000..32a4e6d29 --- /dev/null +++ b/src/lib/documents/__tests__/index-document.test.ts @@ -0,0 +1,230 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; + +/** + * The AI-first / local-fallback content-index decision tree. Pins: a usable + * provider takes the vision path; a text-layer PDF the provider cannot read + * falls through to local; no provider / no consent / budget-out / provider + * error all fall through to the free local path; decrypt + not-found fail + * closed. No plaintext is asserted at the upsert boundary (source + ciphertext + * path is the content-index module's concern, mocked here). + */ + +vi.mock("@/lib/documents/ai-route-support", () => ({ + loadOwnedDocument: vi.fn(), + prepareVisionInput: vi.fn(), +})); +vi.mock("@/lib/documents/content-index", () => ({ + upsertContentIndex: vi.fn().mockResolvedValue({ tokenCount: 5 }), +})); +vi.mock("@/lib/documents/describe", () => ({ + transcribeDocument: vi.fn(), +})); +vi.mock("@/lib/documents/local-extract", () => ({ + localExtractText: vi.fn(), +})); +vi.mock("@/lib/documents/store", () => ({ + decryptDocumentContent: vi.fn(() => Buffer.from([1, 2, 3])), +})); +vi.mock("@/lib/labs/ocr-upload", () => ({ + detectOcrMimeType: vi.fn(() => "application/pdf"), +})); +vi.mock("@/lib/labs/ocr-capability", () => ({ + resolveVisionProvider: vi.fn(), +})); +vi.mock("@/lib/ai/consent-guard", () => ({ + assertConsentForChain: vi.fn().mockResolvedValue(undefined), + ConsentRequiredError: class ConsentRequiredError extends Error {}, +})); +vi.mock("@/lib/ai/coach/budget", () => ({ + buildDateKey: vi.fn(() => "2026-07-07"), + reserveBudget: vi.fn(), + reconcileSpend: vi.fn().mockResolvedValue(undefined), + resolveDailyCap: vi.fn(() => 1000), +})); +vi.mock("@/lib/ai/ai-budgets", () => ({ + AI_BUDGETS: { documentTranscribe: { temperature: 0, maxTokens: 4000 } }, +})); + +import { indexDocumentContent } from "../index-document"; +import { + loadOwnedDocument, + prepareVisionInput, +} from "@/lib/documents/ai-route-support"; +import { upsertContentIndex } from "@/lib/documents/content-index"; +import { transcribeDocument } from "@/lib/documents/describe"; +import { localExtractText } from "@/lib/documents/local-extract"; +import { decryptDocumentContent } from "@/lib/documents/store"; +import { resolveVisionProvider } from "@/lib/labs/ocr-capability"; +import { + assertConsentForChain, + ConsentRequiredError, +} from "@/lib/ai/consent-guard"; +import { reserveBudget, reconcileSpend } from "@/lib/ai/coach/budget"; + +const DOC = { + id: "doc-1", + kind: "OTHER", + contentEncrypted: new Uint8Array([1, 2, 3]), + contentCodec: "binary2", + mimeType: "application/pdf", + status: "STORED", +}; + +const PICK = { + chain: [{ providerType: "anthropic", instance: {} }], + pick: { + entry: { providerType: "anthropic", instance: {} }, + providerType: "anthropic", + pdfSupported: true, + }, +}; + +function visionOk() { + vi.mocked(prepareVisionInput).mockReturnValue({ + ok: true, + images: [], + documents: [{ mediaType: "application/pdf", dataBase64: "AA==" }], + } as never); +} + +beforeEach(() => { + vi.clearAllMocks(); + vi.mocked(loadOwnedDocument).mockResolvedValue(DOC as never); + vi.mocked(reserveBudget).mockResolvedValue({ + allowed: true, + reserved: 1, + } as never); + vi.mocked(localExtractText).mockResolvedValue({ + ok: true, + text: "glucose fasting creatinine values report", + source: "local-pdf", + } as never); +}); + +describe("indexDocumentContent — provider-first path", () => { + it("indexes via the provider (vision) when configured + consented", async () => { + vi.mocked(resolveVisionProvider).mockResolvedValue(PICK as never); + visionOk(); + vi.mocked(transcribeDocument).mockResolvedValue({ + text: "haemoglobin 14.2 cholesterol 190", + } as never); + + const outcome = await indexDocumentContent("user-1", "doc-1"); + expect(outcome).toEqual({ indexed: true, source: "vision", tokenCount: 5 }); + expect(transcribeDocument).toHaveBeenCalledTimes(1); + expect(upsertContentIndex).toHaveBeenCalledWith( + expect.objectContaining({ source: "vision", providerType: "anthropic" }), + ); + // Budget reserved then reconciled at the full reserved amount (charged). + expect(reconcileSpend).toHaveBeenCalledWith("user-1", 1, 1, "2026-07-07"); + // The local path is never touched when the provider succeeds. + expect(localExtractText).not.toHaveBeenCalled(); + }); +}); + +describe("indexDocumentContent — local fallback path", () => { + it("falls back to local when NO provider is configured", async () => { + vi.mocked(resolveVisionProvider).mockResolvedValue({ + chain: [], + pick: null, + } as never); + + const outcome = await indexDocumentContent("user-1", "doc-1"); + expect(outcome).toEqual({ + indexed: true, + source: "local-pdf", + tokenCount: 5, + }); + expect(transcribeDocument).not.toHaveBeenCalled(); + expect(upsertContentIndex).toHaveBeenCalledWith( + expect.objectContaining({ source: "local-pdf", providerType: null }), + ); + }); + + it("falls back to local when consent is not granted", async () => { + vi.mocked(resolveVisionProvider).mockResolvedValue(PICK as never); + vi.mocked(assertConsentForChain).mockRejectedValueOnce( + new ConsentRequiredError("insights" as never), + ); + + const outcome = await indexDocumentContent("user-1", "doc-1"); + expect(outcome).toMatchObject({ indexed: true, source: "local-pdf" }); + expect(transcribeDocument).not.toHaveBeenCalled(); + }); + + it("falls back to local when the provider cannot read the PDF", async () => { + vi.mocked(resolveVisionProvider).mockResolvedValue(PICK as never); + vi.mocked(prepareVisionInput).mockReturnValue({ + ok: false, + reason: "pdfNeedsAnthropic", + } as never); + + const outcome = await indexDocumentContent("user-1", "doc-1"); + expect(outcome).toMatchObject({ indexed: true, source: "local-pdf" }); + expect(reserveBudget).not.toHaveBeenCalled(); + expect(transcribeDocument).not.toHaveBeenCalled(); + }); + + it("falls back to local when the daily budget is exhausted", async () => { + vi.mocked(resolveVisionProvider).mockResolvedValue(PICK as never); + visionOk(); + vi.mocked(reserveBudget).mockResolvedValue({ + allowed: false, + reserved: 0, + } as never); + + const outcome = await indexDocumentContent("user-1", "doc-1"); + expect(outcome).toMatchObject({ indexed: true, source: "local-pdf" }); + expect(transcribeDocument).not.toHaveBeenCalled(); + }); + + it("refunds and falls back to local when the provider call throws", async () => { + vi.mocked(resolveVisionProvider).mockResolvedValue(PICK as never); + visionOk(); + vi.mocked(transcribeDocument).mockRejectedValueOnce(new Error("boom")); + + const outcome = await indexDocumentContent("user-1", "doc-1"); + expect(outcome).toMatchObject({ indexed: true, source: "local-pdf" }); + // Reservation refunded to zero spend. + expect(reconcileSpend).toHaveBeenCalledWith("user-1", 1, 0, "2026-07-07"); + expect(localExtractText).toHaveBeenCalledTimes(1); + }); + + it("reports local-empty for a scanned PDF with no usable provider", async () => { + vi.mocked(resolveVisionProvider).mockResolvedValue({ + chain: [], + pick: null, + } as never); + vi.mocked(localExtractText).mockResolvedValue({ + ok: false, + reason: "empty", + } as never); + + const outcome = await indexDocumentContent("user-1", "doc-1"); + expect(outcome).toEqual({ indexed: false, reason: "local-empty" }); + expect(upsertContentIndex).not.toHaveBeenCalled(); + }); +}); + +describe("indexDocumentContent — fail-closed", () => { + it("returns not-found for an unknown / non-owned document", async () => { + vi.mocked(loadOwnedDocument).mockResolvedValue(null as never); + const outcome = await indexDocumentContent("user-1", "missing"); + expect(outcome).toEqual({ indexed: false, reason: "not-found" }); + expect(resolveVisionProvider).not.toHaveBeenCalled(); + }); + + it("fails closed to decrypt-error when local decryption throws", async () => { + vi.mocked(resolveVisionProvider).mockResolvedValue({ + chain: [], + pick: null, + } as never); + vi.mocked(decryptDocumentContent).mockImplementationOnce(() => { + throw new Error("bad key"); + }); + + const outcome = await indexDocumentContent("user-1", "doc-1"); + expect(outcome).toEqual({ indexed: false, reason: "decrypt-error" }); + expect(upsertContentIndex).not.toHaveBeenCalled(); + }); +}); diff --git a/src/lib/documents/__tests__/local-extract.test.ts b/src/lib/documents/__tests__/local-extract.test.ts new file mode 100644 index 000000000..fc641ea34 --- /dev/null +++ b/src/lib/documents/__tests__/local-extract.test.ts @@ -0,0 +1,82 @@ +import { describe, expect, it, vi } from "vitest"; +import { jsPDF } from "jspdf"; + +/** + * Local, provider-free text extraction. Pins: a real text-layer PDF yields its + * text with source "local-pdf" (no provider, no OCR); a PDF below the min-chars + * gate is treated as empty (scanned); an image is the deferred-OCR seam + * (unsupported); malformed bytes fail closed to "error" without throwing. + */ + +vi.mock("@/lib/logging/context", () => ({ annotate: vi.fn() })); + +import { + extractPdfText, + localExtractText, + LOCAL_TEXT_MIN_CHARS, +} from "../local-extract"; + +/** Build a real text-layer PDF (jsPDF is isomorphic; Helvetica text layer). */ +function textLayerPdf(lines: string[]): Buffer { + const doc = new jsPDF(); + let y = 10; + for (const line of lines) { + doc.text(line, 10, y); + y += 10; + } + return Buffer.from(doc.output("arraybuffer")); +} + +describe("extractPdfText", () => { + it("reads a text-layer PDF's embedded text as source local-pdf", async () => { + const pdf = textLayerPdf([ + "Haemoglobin 14.2 g/dL Cholesterol total 190 mg/dL", + "Patient report creatinine glucose fasting values", + ]); + const result = await extractPdfText(pdf); + expect(result.ok).toBe(true); + if (result.ok) { + expect(result.source).toBe("local-pdf"); + expect(result.text).toContain("Haemoglobin"); + expect(result.text).toContain("creatinine"); + // The default "-- N of M --" page marker must not leak into the text. + expect(result.text).not.toMatch(/-- \d+ of \d+ --/); + } + }); + + it("treats a PDF below the min-chars gate as empty (scanned proxy)", async () => { + const pdf = textLayerPdf(["x"]); + expect("x".length).toBeLessThan(LOCAL_TEXT_MIN_CHARS); + const result = await extractPdfText(pdf); + expect(result).toEqual({ ok: false, reason: "empty" }); + }); + + it("fails closed to error on malformed bytes without throwing", async () => { + const result = await extractPdfText(Buffer.from([1, 2, 3, 4, 5, 6, 7, 8])); + expect(result).toEqual({ ok: false, reason: "error" }); + }); +}); + +describe("localExtractText", () => { + it("routes application/pdf through the text-layer reader", async () => { + const pdf = textLayerPdf([ + "Discharge letter diagnosis medication dosage instructions", + ]); + const result = await localExtractText(pdf, "application/pdf"); + expect(result.ok).toBe(true); + if (result.ok) expect(result.source).toBe("local-pdf"); + }); + + it("returns unsupported for images (deferred server-OCR seam)", async () => { + const result = await localExtractText( + Buffer.from([0xff, 0xd8]), + "image/jpeg", + ); + expect(result).toEqual({ ok: false, reason: "unsupported" }); + }); + + it("returns unsupported for any other MIME", async () => { + const result = await localExtractText(Buffer.from([0]), "text/plain"); + expect(result).toEqual({ ok: false, reason: "unsupported" }); + }); +}); diff --git a/src/lib/documents/content-index.ts b/src/lib/documents/content-index.ts index 964f6a1c3..76a8b98d1 100644 --- a/src/lib/documents/content-index.ts +++ b/src/lib/documents/content-index.ts @@ -240,8 +240,17 @@ export function decryptIndexText(buf: Uint8Array): string { return decryptFromBytes(buf); } -/** Provenance of the indexed text. */ -export type ContentIndexSource = "vision" | "text-ocr"; +/** + * Provenance of the indexed text. + * - `vision` → an AI provider transcribed the stored original (image or PDF, + * incl. scanned) — the AI-first primary path. + * - `text-ocr` → browser-OCR text posted by the client (opt-in local OCR). + * - `local-pdf` → server-side text-layer extraction of a PDF (`pdf-parse`), no + * provider, no egress — the fallback when no provider is usable. + * - `local-ocr` → server-side OCR of a scanned image/PDF (deferred follow-up). + */ +export type ContentIndexSource = + "vision" | "text-ocr" | "local-pdf" | "local-ocr"; export interface UpsertContentIndexArgs { userId: string; diff --git a/src/lib/documents/index-document.ts b/src/lib/documents/index-document.ts new file mode 100644 index 000000000..99f673cd9 --- /dev/null +++ b/src/lib/documents/index-document.ts @@ -0,0 +1,233 @@ +/** + * Content-index ONE stored document — the shared, AI-first decision tree behind + * both the per-document auto-index-on-upload job and the corpus backfill. + * + * Ordering (maintainer, 2026-07-07 — AI-first, local as fallback): + * 1. PROVIDER (primary). When a vision-capable AI provider is configured AND + * the user has consented, decrypt the stored original and run ONE provider + * transcription — rich, handles scanned PDFs + images via vision. Budget- + * reserved + reconciled exactly like the interactive index route; owner- + * scoped throughout. Codex-over-OAuth is subscription-covered, so this + * path carries no per-token API bill (see the research note). + * 2. LOCAL (fallback). When no provider is usable — none configured, consent + * not granted, the provider can't read this file (e.g. a text-layer PDF on + * a non-Anthropic account), the daily budget is reached, or the provider + * call fails — fall back to server-side text-layer extraction (`pdf-parse`, + * no egress, milliseconds). So content search works at a baseline even with + * no AI, and every text-layer PDF is searchable regardless of provider. + * + * Both paths write the SAME blind, encrypted index via `upsertContentIndex` + * (AES-256-GCM text + opaque HMAC token tags) — nothing readable at rest. The + * function NEVER throws for an expected outcome; it returns a tagged result the + * caller logs, so a bad document can never abort an upload or a batch. + */ +import { Buffer } from "node:buffer"; + +import { AI_BUDGETS } from "@/lib/ai/ai-budgets"; +import { + assertConsentForChain, + ConsentRequiredError, +} from "@/lib/ai/consent-guard"; +import { + buildDateKey, + reconcileSpend, + reserveBudget, + resolveDailyCap, +} from "@/lib/ai/coach/budget"; +import { + loadOwnedDocument, + prepareVisionInput, + type LoadedDocument, +} from "@/lib/documents/ai-route-support"; +import { + upsertContentIndex, + type ContentIndexSource, +} from "@/lib/documents/content-index"; +import { transcribeDocument } from "@/lib/documents/describe"; +import { localExtractText } from "@/lib/documents/local-extract"; +import { decryptDocumentContent } from "@/lib/documents/store"; +import { detectOcrMimeType } from "@/lib/labs/ocr-upload"; +import type { ProviderChainResolved } from "@/lib/ai/provider-runner"; +import { resolveVisionProvider } from "@/lib/labs/ocr-capability"; + +/** + * Outcome of one document's index attempt. A provider that is not usable (none + * configured, no consent, budget exhausted, cannot read the file, or errored) + * is never itself a terminal outcome — the tree always falls through to the + * free local path, so the terminal reasons are only the local ones. + */ +export type IndexOutcome = + | { indexed: true; source: ContentIndexSource; tokenCount: number } + | { + indexed: false; + reason: + "not-found" | "local-empty" | "local-unsupported" | "decrypt-error"; + }; + +/** + * A resolved, consent-checked provider context. Resolve ONCE per user and reuse + * across a batch so the corpus backfill does not re-read the provider chain per + * document. `usable` is true only when a vision pick exists AND consent passed. + */ +export interface ResolvedIndexProvider { + chain: ProviderChainResolved[]; + pick: Awaited>["pick"]; + consentOk: boolean; + dailyCap: number; +} + +/** + * Resolve the user's vision provider and pre-check consent once. A missing pick + * or an un-granted consent both yield an unusable context — the caller then + * falls straight to the local path. A NON-consent error propagates. + */ +export async function resolveIndexProvider( + userId: string, +): Promise { + const { chain, pick } = await resolveVisionProvider(userId); + let consentOk = false; + if (pick) { + try { + await assertConsentForChain({ userId, chain, surface: "insights" }); + consentOk = true; + } catch (err) { + if (!(err instanceof ConsentRequiredError)) throw err; + } + } + const dailyCap = pick + ? resolveDailyCap([{ providerType: pick.entry.providerType }]) + : 0; + return { chain, pick, consentOk, dailyCap }; +} + +/** + * Try the PROVIDER (vision) path for one already-loaded document. Returns a + * terminal outcome on success/budget/provider-error, or `null` to signal + * "fall through to local" (no usable provider, or the provider cannot read this + * particular file — e.g. a PDF on a non-Anthropic account). + */ +async function tryProviderIndex( + userId: string, + document: LoadedDocument, + provider: ResolvedIndexProvider, +): Promise { + const { pick } = provider; + if (!pick || !provider.consentOk) return null; + + const vision = prepareVisionInput(document, pick.pdfSupported); + if (!vision.ok) { + // A decrypt failure is terminal (local would fail the same way); anything + // else (fileType / pdfNeedsAnthropic) falls through to the local path — a + // text-layer PDF on a non-Anthropic account is read locally for free. + if (vision.reason === "decryptFailed") { + return { indexed: false, reason: "decrypt-error" }; + } + return null; + } + + const dateKey = buildDateKey(); + const reservation = await reserveBudget( + userId, + AI_BUDGETS.documentTranscribe.maxTokens, + dateKey, + provider.dailyCap, + ); + // Budget exhausted → fall through to the free local path rather than stall; + // a text-layer PDF stays searchable even once the AI allowance is spent. + if (!reservation.allowed) return null; + + try { + const { text } = await transcribeDocument({ + provider: pick.entry.instance, + providerType: pick.providerType, + images: vision.images, + documents: vision.documents, + }); + await reconcileSpend( + userId, + reservation.reserved, + reservation.reserved, + dateKey, + ); + const { tokenCount } = await upsertContentIndex({ + userId, + documentId: document.id, + text, + source: "vision", + providerType: pick.providerType, + }); + return { indexed: true, source: "vision", tokenCount }; + } catch { + // Refund the reservation and let the caller fall through to local — a + // transient provider miss must never leave a text-layer PDF unsearchable. + await reconcileSpend(userId, reservation.reserved, 0, dateKey); + return null; + } +} + +/** Try the LOCAL (provider-free) text-layer path for one loaded document. */ +async function tryLocalIndex( + userId: string, + document: LoadedDocument, +): Promise { + let buffer: Buffer; + try { + buffer = decryptDocumentContent( + document.contentEncrypted, + document.contentCodec, + ); + } catch { + return { indexed: false, reason: "decrypt-error" }; + } + + // Re-derive the MIME from the bytes (never trust the stored label), matching + // the provider path's posture; fall back to the stored MIME if unrecognised. + const mime = detectOcrMimeType(buffer) ?? document.mimeType; + const result = await localExtractText(buffer, mime); + if (result.ok) { + const { tokenCount } = await upsertContentIndex({ + userId, + documentId: document.id, + text: result.text, + source: result.source, + providerType: null, + }); + return { indexed: true, source: result.source, tokenCount }; + } + if (result.reason === "unsupported") { + return { indexed: false, reason: "local-unsupported" }; + } + if (result.reason === "error") { + return { indexed: false, reason: "decrypt-error" }; + } + return { indexed: false, reason: "local-empty" }; +} + +/** + * Index one already-loaded document: provider-first, local-fallback. Reuses a + * pre-resolved provider context so a batch resolves the chain once. + */ +export async function indexLoadedDocument( + userId: string, + document: LoadedDocument, + provider: ResolvedIndexProvider, +): Promise { + const providerOutcome = await tryProviderIndex(userId, document, provider); + if (providerOutcome) return providerOutcome; + return tryLocalIndex(userId, document); +} + +/** + * Index ONE document by id (owner-scoped): resolve the provider, load the + * document, then run the provider-first/local-fallback tree. This is the entry + * point the per-document auto-index-on-upload job calls. + */ +export async function indexDocumentContent( + userId: string, + documentId: string, +): Promise { + const document = await loadOwnedDocument(userId, documentId); + if (!document) return { indexed: false, reason: "not-found" }; + const provider = await resolveIndexProvider(userId); + return indexLoadedDocument(userId, document, provider); +} diff --git a/src/lib/documents/local-extract.ts b/src/lib/documents/local-extract.ts new file mode 100644 index 000000000..e87aa2d96 --- /dev/null +++ b/src/lib/documents/local-extract.ts @@ -0,0 +1,116 @@ +/** + * Local (provider-free) text extraction for the document vault. + * + * When NO AI provider is configured — or the user has not consented to one — + * content search still needs a baseline. This module extracts a PDF's embedded + * text layer SERVER-SIDE with `pdf-parse` (already a direct dependency; its + * `getText()` touches only the pure-JS/wasm pdfjs core, never the native + * `@napi-rs/canvas`), in milliseconds, with no OCR and no third-party egress. + * The extracted text feeds the SAME blind, encrypted content index the provider + * path writes (`upsertContentIndex`), so nothing readable is stored at rest. + * + * Digitally-generated PDFs (lab reports, discharge letters, referrals — the + * clinical common case) carry a real embedded text layer this path reads. + * Scanned/image-only PDFs and uploaded images have NO text layer: local OCR + * (server-side tesseract + `deu+eng` traineddata baked into the image, plus + * `@napi-rs/canvas` rasterisation for scanned PDFs) is a deferred follow-up, NOT + * built here — it carries the only real Alpine/standalone-tracing + CPU cost. + * Until it lands, those documents stay un-indexed on the local path and are + * covered by the provider (vision) path whenever one is configured. + */ +import { Buffer } from "node:buffer"; + +import { PDFParse } from "pdf-parse"; + +import { annotate } from "@/lib/logging/context"; + +/** + * Minimum joined text length for a PDF to count as a real text-layer document. + * Below this a PDF is treated as scanned/empty — a few stray glyphs from a form + * field or a page number are not a usable index. + */ +export const LOCAL_TEXT_MIN_CHARS = 24; + +/** Cap on pages parsed — bounds CPU/time on a pathological many-page PDF. */ +const LOCAL_MAX_PAGES = 200; + +/** Provenance tag written to `DocumentContentIndex.source` on the local path. */ +export type LocalExtractSource = "local-pdf" | "local-ocr"; + +/** + * Outcome of a local extraction attempt. + * - `ok` → usable text was recovered (index it). + * - `empty` → a valid document with no usable text layer (scanned PDF); + * a provider vision pass is the only way to read it. + * - `unsupported` → a MIME the local path cannot read without OCR (images) or + * at all; leave un-indexed locally. + * - `error` → the file was malformed / unreadable; treat as un-indexed. + */ +export type LocalExtractResult = + | { ok: true; text: string; source: LocalExtractSource } + | { ok: false; reason: "empty" | "unsupported" | "error" }; + +/** + * Extract a text-layer PDF's embedded text server-side. Returns `empty` for a + * scanned/no-text-layer PDF and `error` for a malformed/encrypted one — never + * throws, so a bad document can never abort the caller (the upload/backfill job + * just leaves it un-indexed). + */ +export async function extractPdfText( + buffer: Buffer, +): Promise { + let parser: PDFParse | null = null; + try { + parser = new PDFParse({ data: new Uint8Array(buffer) }); + const result = await parser.getText({ + // Bound the page walk and drop the default "-- N of M --" page markers so + // only real document text is tokenised. + first: LOCAL_MAX_PAGES, + pageJoiner: "", + }); + const text = (result.text ?? "").trim(); + if (text.length < LOCAL_TEXT_MIN_CHARS) { + return { ok: false, reason: "empty" }; + } + return { ok: true, text, source: "local-pdf" }; + } catch { + return { ok: false, reason: "error" }; + } finally { + if (parser) { + try { + await parser.destroy(); + } catch { + // Best-effort teardown; a destroy failure must not mask the result. + } + } + } +} + +/** + * Dispatch local extraction by MIME. PDFs go through the text-layer reader; + * images are the deferred server-OCR seam (see the module header) and return + * `unsupported` so the caller falls through to the provider path or leaves the + * document un-indexed. + */ +export async function localExtractText( + buffer: Buffer, + mime: string, +): Promise { + if (mime === "application/pdf") { + return extractPdfText(buffer); + } + if (mime.startsWith("image/")) { + // SEAM (NOT DONE): server-side tesseract OCR for scanned images/PDFs is a + // deferred follow-up. It needs the `deu+eng` traineddata baked into the + // image and (for scanned PDFs) `@napi-rs/canvas` traced into the standalone + // build — the only real deployment cost in this area. Until it ships, a + // no-provider image stays un-indexed locally; a configured vision provider + // covers it on the AI-first path. + annotate({ + action: { name: "documents.localIndex.ocrNotImplemented" }, + meta: { mime }, + }); + return { ok: false, reason: "unsupported" }; + } + return { ok: false, reason: "unsupported" }; +} diff --git a/src/lib/jobs/__tests__/document-index.test.ts b/src/lib/jobs/__tests__/document-index.test.ts new file mode 100644 index 000000000..3de576f36 --- /dev/null +++ b/src/lib/jobs/__tests__/document-index.test.ts @@ -0,0 +1,63 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; + +/** + * Per-document auto-index job. Pins: the enqueue coalesces per document via a + * singletonKey and no-ops when the boss is absent; the worker delegates to the + * shared decision tree and never throws for an expected outcome. + */ + +vi.mock("@/lib/documents/index-document", () => ({ + indexDocumentContent: vi.fn(), +})); +vi.mock("@/lib/jobs/boss-instance", () => ({ getGlobalBoss: vi.fn() })); +vi.mock("@/lib/logging/context", () => ({ annotate: vi.fn() })); + +import { + DOCUMENT_INDEX_QUEUE, + enqueueDocumentIndex, + runDocumentIndex, +} from "../document-index"; +import { indexDocumentContent } from "@/lib/documents/index-document"; +import { getGlobalBoss } from "@/lib/jobs/boss-instance"; + +beforeEach(() => { + vi.clearAllMocks(); +}); + +describe("enqueueDocumentIndex", () => { + it("coalesces per document via a singletonKey", async () => { + const send = vi.fn().mockResolvedValue("job-1"); + vi.mocked(getGlobalBoss).mockReturnValue({ send } as never); + + const result = await enqueueDocumentIndex("user-1", "doc-1"); + expect(result).toEqual({ enqueued: true }); + expect(send).toHaveBeenCalledWith( + DOCUMENT_INDEX_QUEUE, + expect.objectContaining({ userId: "user-1", documentId: "doc-1" }), + expect.objectContaining({ singletonKey: "document-index|doc-1" }), + ); + }); + + it("no-ops when the boss is not running", async () => { + vi.mocked(getGlobalBoss).mockReturnValue(null as never); + const result = await enqueueDocumentIndex("user-1", "doc-1"); + expect(result).toEqual({ enqueued: false }); + }); +}); + +describe("runDocumentIndex", () => { + it("delegates to the shared decision tree", async () => { + vi.mocked(indexDocumentContent).mockResolvedValue({ + indexed: true, + source: "local-pdf", + tokenCount: 3, + } as never); + await runDocumentIndex({ userId: "user-1", documentId: "doc-1" }); + expect(indexDocumentContent).toHaveBeenCalledWith("user-1", "doc-1"); + }); + + it("ignores a payload missing ids without calling the tree", async () => { + await runDocumentIndex({ userId: "", documentId: "" }); + expect(indexDocumentContent).not.toHaveBeenCalled(); + }); +}); diff --git a/src/lib/jobs/document-index.ts b/src/lib/jobs/document-index.ts new file mode 100644 index 000000000..bc4396759 --- /dev/null +++ b/src/lib/jobs/document-index.ts @@ -0,0 +1,72 @@ +/** + * Automatic per-document content indexing, enqueued the moment a document is + * stored (`POST /api/documents/inbound`). Fire-and-forget: the upload response + * never blocks on or fails because of indexing. + * + * The job runs the shared AI-first decision tree (`indexDocumentContent`): + * provider (vision) first when one is configured + consented, else local + * text-layer extraction. Owner-scoped; the provider path is consent + budget + * gated; the local path needs neither (no egress). Serial concurrency keeps the + * provider calls + PDF parsing off the request pool. + * + * The queue MUST be registered in the maintenance registrar + * (`src/lib/jobs/reminder/register-maintenance.ts`) so pg-boss provisions it. + */ +import { indexDocumentContent } from "@/lib/documents/index-document"; +import { getGlobalBoss } from "@/lib/jobs/boss-instance"; +import { annotate } from "@/lib/logging/context"; + +export const DOCUMENT_INDEX_QUEUE = "document-index"; + +/** Serial concurrency — provider calls + PDF parse, kept off the request pool. */ +export const DOCUMENT_INDEX_CONCURRENCY = 1; + +export interface DocumentIndexPayload { + userId: string; + documentId: string; + enqueuedAt?: string; +} + +/** + * Index one document (owner-scoped, provider-first/local-fallback). Never + * throws for an expected outcome — the decision tree returns a tagged result — + * so pg-boss does not retry a document that simply has no indexable text. + */ +export async function runDocumentIndex( + payload: DocumentIndexPayload, +): Promise { + const { userId, documentId } = payload; + if (!userId || !documentId) return; + const outcome = await indexDocumentContent(userId, documentId); + annotate({ + action: { name: "documents.autoIndex.run" }, + meta: outcome.indexed + ? { documentId, indexed: true, source: outcome.source } + : { documentId, indexed: false, reason: outcome.reason }, + }); +} + +/** + * Enqueue a per-document index job. Coalesced by `singletonKey` per document so + * a duplicate/idempotent re-upload cannot pile up parallel runs. Fire-and- + * forget — a missing boss (worker not up) is a no-op, never an upload failure. + */ +export async function enqueueDocumentIndex( + userId: string, + documentId: string, +): Promise<{ enqueued: boolean }> { + const boss = getGlobalBoss(); + if (!boss) return { enqueued: false }; + const payload: DocumentIndexPayload = { + userId, + documentId, + enqueuedAt: new Date().toISOString(), + }; + const jobId = await boss.send(DOCUMENT_INDEX_QUEUE, payload, { + retryLimit: 2, + retryDelay: 30, + retryBackoff: true, + singletonKey: `document-index|${documentId}`, + }); + return { enqueued: Boolean(jobId) }; +} diff --git a/src/lib/jobs/reminder/register-maintenance.ts b/src/lib/jobs/reminder/register-maintenance.ts index f44868e2b..ccdad6fc2 100644 --- a/src/lib/jobs/reminder/register-maintenance.ts +++ b/src/lib/jobs/reminder/register-maintenance.ts @@ -75,6 +75,12 @@ import { runContentIndexBackfillForUser, type ContentIndexBackfillPayload, } from "@/lib/jobs/document-content-index-backfill"; +import { + DOCUMENT_INDEX_QUEUE, + DOCUMENT_INDEX_CONCURRENCY, + runDocumentIndex, + type DocumentIndexPayload, +} from "@/lib/jobs/document-index"; import { ENCRYPTION_KEY_ROTATE_QUEUE, ENCRYPTION_KEY_ROTATE_CONCURRENCY, @@ -310,6 +316,11 @@ const allQueues = [ // Without this entry pg-boss never provisions the queue and the trigger // silently no-ops. CONTENT_INDEX_BACKFILL_QUEUE, + // Document AI — automatic per-document content indexing, enqueued on upload. + // Provider-first (vision) when configured + consented, else local text-layer + // extraction. Without this entry pg-boss never provisions the queue and every + // upload enqueue silently no-ops. + DOCUMENT_INDEX_QUEUE, ]; const schedules: ScheduleEntry[] = [ @@ -682,6 +693,32 @@ export async function registerMaintenanceQueues( }, ); + // Document AI — automatic per-document content indexing. Enqueued on upload + // (one job per stored document); provider-first (vision) when configured + + // consented, else local text-layer extraction. Serial concurrency so the + // provider calls + PDF parse never crowd the request pool. + await boss.work( + DOCUMENT_INDEX_QUEUE, + { localConcurrency: DOCUMENT_INDEX_CONCURRENCY }, + async (jobs) => { + for (const job of jobs) { + const { userId, documentId } = job.data; + if (!userId || !documentId) continue; + try { + await runDocumentIndex(job.data); + } catch (err) { + recordError(); + workerLog( + "error", + `[document-index] user=${userId} document=${documentId} failed`, + err, + ); + throw err; + } + } + }, + ); + // v1.25 (W-ENV) — nightly environment fetch. The daily discovery tick (empty // payload) fans out one per-user job per opted-in account; the queue also // serves the on-demand backfill payloads from the settings surface. Serial