From 8471e0edc923bbcacfedb7c508912ecc206e3780 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Sun, 25 Oct 2026 15:56:31 +0100 Subject: [PATCH] fix(documents): say a PDF past the reader's limits is too large to read A text read that crossed the isolated reader's memory or time bound came back as a plain read error, which the index job records as decrypt-error, so the document said its stored file could not be opened. The local extractor now answers resource-limit for PdfResourceLimitError, the index records it under that name, and the detail view says the PDF is too large or complex to read here, in every locale. Like the thumbnail's refusal of the same bytes it is final: nothing retries it on its own. --- docs/api/openapi.yaml | 2 + messages/de.json | 3 +- messages/en.json | 3 +- messages/es.json | 3 +- messages/fr.json | 3 +- messages/it.json | 3 +- messages/ko.json | 3 +- messages/nl.json | 3 +- messages/pl.json | 3 +- messages/pt-BR.json | 3 +- prisma/schema.prisma | 8 +-- .../__tests__/index-document.test.ts | 30 +++++++++++ .../local-extract-resource-limit.test.ts | 53 +++++++++++++++++++ src/lib/documents/index-document.ts | 7 +++ src/lib/documents/local-extract.ts | 23 ++++++-- src/lib/openapi/routes/documents.ts | 1 + src/lib/validations/inbound-documents.ts | 1 + 17 files changed, 136 insertions(+), 16 deletions(-) create mode 100644 src/lib/documents/__tests__/local-extract-resource-limit.test.ts diff --git a/docs/api/openapi.yaml b/docs/api/openapi.yaml index 7f5523d68..02f8a3095 100644 --- a/docs/api/openapi.yaml +++ b/docs/api/openapi.yaml @@ -79798,6 +79798,7 @@ components: - local-empty - local-unsupported - decrypt-error + - resource-limit - raster-failed - provider-error - pdf-needs-anthropic @@ -80370,6 +80371,7 @@ components: - local-empty - local-unsupported - decrypt-error + - resource-limit - raster-failed - provider-error - pdf-needs-anthropic diff --git a/messages/de.json b/messages/de.json index bb2624eeb..96a78eff5 100644 --- a/messages/de.json +++ b/messages/de.json @@ -11877,7 +11877,8 @@ "raster-failed": "Die PDF-Seiten konnten zum Lesen nicht gerendert werden. Gescannte Fax-Kompressionsformate sind eine bekannte Ursache.", "provider-error": "Der KI-Anbieter konnte dieses Dokument beim letzten Versuch nicht lesen.", "pdf-needs-anthropic": "Der eingerichtete KI-Anbieter kann keine PDF-Dateien lesen. Ein Claude-Anbieter oder lokale OCR kann es.", - "empty-transcription": "Der KI-Anbieter hat für dieses Dokument keinen Text geliefert." + "empty-transcription": "Der KI-Anbieter hat für dieses Dokument keinen Text geliefert.", + "resource-limit": "Dieses PDF ist zu groß oder zu komplex, um es hier zu lesen. Die Datei selbst ist in Ordnung und lässt sich weiterhin öffnen." } }, "loadError": "Dieses Dokument konnte nicht geladen werden.", diff --git a/messages/en.json b/messages/en.json index f99c0d281..bb4cf34c4 100644 --- a/messages/en.json +++ b/messages/en.json @@ -11877,7 +11877,8 @@ "raster-failed": "The PDF pages couldn't be rendered for reading. Scanned fax compression formats are a known cause.", "provider-error": "The AI provider couldn't read this document on the last try.", "pdf-needs-anthropic": "The configured AI provider can't read PDF files. A Claude provider or local OCR can.", - "empty-transcription": "The AI provider returned no text for this document." + "empty-transcription": "The AI provider returned no text for this document.", + "resource-limit": "This PDF is too large or complex to read here. The file itself is fine and still opens." } }, "loadError": "Couldn't load this document.", diff --git a/messages/es.json b/messages/es.json index 37bf14e28..5c9056ae2 100644 --- a/messages/es.json +++ b/messages/es.json @@ -11877,7 +11877,8 @@ "raster-failed": "Las páginas del PDF no se pudieron procesar para su lectura. Los formatos de compresión de fax escaneados son una causa conocida.", "provider-error": "El proveedor de IA no pudo leer este documento en el último intento.", "pdf-needs-anthropic": "El proveedor de IA configurado no puede leer archivos PDF. Un proveedor Claude o el OCR local sí pueden.", - "empty-transcription": "El proveedor de IA no devolvió texto para este documento." + "empty-transcription": "El proveedor de IA no devolvió texto para este documento.", + "resource-limit": "Este PDF es demasiado grande o complejo para leerlo aquí. El archivo en sí está bien y se sigue pudiendo abrir." } }, "loadError": "No se pudo cargar este documento.", diff --git a/messages/fr.json b/messages/fr.json index 9afb8acda..ddc754baf 100644 --- a/messages/fr.json +++ b/messages/fr.json @@ -11877,7 +11877,8 @@ "raster-failed": "Les pages du PDF n’ont pas pu être rendues pour la lecture. Les formats de compression de fax numérisés en sont une cause connue.", "provider-error": "Le fournisseur d’IA n’a pas pu lire ce document lors de la dernière tentative.", "pdf-needs-anthropic": "Le fournisseur d’IA configuré ne peut pas lire les fichiers PDF. Un fournisseur Claude ou l’OCR local le peut.", - "empty-transcription": "Le fournisseur d’IA n’a renvoyé aucun texte pour ce document." + "empty-transcription": "Le fournisseur d’IA n’a renvoyé aucun texte pour ce document.", + "resource-limit": "Ce PDF est trop volumineux ou trop complexe pour être lu ici. Le fichier lui-même est intact et s'ouvre toujours." } }, "loadError": "Impossible de charger ce document.", diff --git a/messages/it.json b/messages/it.json index a67d5d50e..fe6450c8d 100644 --- a/messages/it.json +++ b/messages/it.json @@ -11877,7 +11877,8 @@ "raster-failed": "Le pagine del PDF non possono essere elaborate per la lettura. I formati di compressione fax digitalizzati sono una causa nota.", "provider-error": "Il provider IA non è riuscito a leggere questo documento all’ultimo tentativo.", "pdf-needs-anthropic": "Il provider IA configurato non può leggere i file PDF. Un provider Claude o l’OCR locale sì.", - "empty-transcription": "Il provider IA non ha restituito testo per questo documento." + "empty-transcription": "Il provider IA non ha restituito testo per questo documento.", + "resource-limit": "Questo PDF è troppo grande o complesso per essere letto qui. Il file in sé è a posto e si apre ancora." } }, "loadError": "Impossibile caricare questo documento.", diff --git a/messages/ko.json b/messages/ko.json index 1caf6faf9..2ef2164d3 100644 --- a/messages/ko.json +++ b/messages/ko.json @@ -11877,7 +11877,8 @@ "raster-failed": "PDF 페이지를 읽기용으로 렌더링하지 못했어요. 스캔 팩스 압축 형식이 흔한 원인이에요.", "provider-error": "지난번에 AI 제공자가 이 문서를 읽지 못했어요.", "pdf-needs-anthropic": "설정된 AI 제공자는 PDF 파일을 읽을 수 없어요. Claude 제공자나 로컬 OCR로는 읽을 수 있어요.", - "empty-transcription": "AI 제공자가 이 문서에서 텍스트를 찾지 못했어요." + "empty-transcription": "AI 제공자가 이 문서에서 텍스트를 찾지 못했어요.", + "resource-limit": "이 PDF는 너무 크거나 복잡해서 여기서 읽을 수 없습니다. 파일 자체는 정상이며 계속 열 수 있습니다." } }, "loadError": "이 문서를 불러오지 못했어요.", diff --git a/messages/nl.json b/messages/nl.json index 01901ab56..9bd5560ec 100644 --- a/messages/nl.json +++ b/messages/nl.json @@ -11877,7 +11877,8 @@ "raster-failed": "De PDF-pagina's konden niet worden weergegeven om te lezen. Compressieformaten van gescande faxen zijn een bekende oorzaak.", "provider-error": "De AI-provider kon dit document bij de laatste poging niet lezen.", "pdf-needs-anthropic": "De ingestelde AI-provider kan geen PDF-bestanden lezen. Een Claude-provider of lokale OCR kan dat wel.", - "empty-transcription": "De AI-provider gaf geen tekst terug voor dit document." + "empty-transcription": "De AI-provider gaf geen tekst terug voor dit document.", + "resource-limit": "Deze pdf is te groot of te complex om hier te lezen. Het bestand zelf is in orde en opent nog steeds." } }, "loadError": "Dit document kon niet worden geladen.", diff --git a/messages/pl.json b/messages/pl.json index c498fc660..2a1673bcd 100644 --- a/messages/pl.json +++ b/messages/pl.json @@ -11877,7 +11877,8 @@ "raster-failed": "Nie udało się przetworzyć stron PDF do odczytu. Znaną przyczyną są skany w faksowych formatach kompresji.", "provider-error": "Dostawca AI nie zdołał odczytać tego dokumentu przy ostatniej próbie.", "pdf-needs-anthropic": "Skonfigurowany dostawca AI nie odczytuje plików PDF. Poradzi sobie z tym dostawca Claude lub lokalny OCR.", - "empty-transcription": "Dostawca AI nie zwrócił tekstu dla tego dokumentu." + "empty-transcription": "Dostawca AI nie zwrócił tekstu dla tego dokumentu.", + "resource-limit": "Ten PDF jest zbyt duży lub zbyt złożony, żeby odczytać go tutaj. Sam plik jest w porządku i nadal się otwiera." } }, "loadError": "Nie udało się wczytać tego dokumentu.", diff --git a/messages/pt-BR.json b/messages/pt-BR.json index ab7eaf3ca..cc011e70a 100644 --- a/messages/pt-BR.json +++ b/messages/pt-BR.json @@ -11877,7 +11877,8 @@ "raster-failed": "As páginas do PDF não puderam ser renderizadas para leitura. Formatos de compressão de fax digitalizado são uma causa conhecida.", "provider-error": "O provedor de IA não conseguiu ler este documento na última tentativa.", "pdf-needs-anthropic": "O provedor de IA configurado não consegue ler arquivos PDF. Um provedor Claude ou OCR local consegue.", - "empty-transcription": "O provedor de IA não retornou texto para este documento." + "empty-transcription": "O provedor de IA não retornou texto para este documento.", + "resource-limit": "Este PDF é grande ou complexo demais para ser lido aqui. O arquivo em si está bem e continua abrindo." } }, "loadError": "Não foi possível carregar este documento.", diff --git a/prisma/schema.prisma b/prisma/schema.prisma index 67cfbeb4e..2b5fd6b2c 100644 --- a/prisma/schema.prisma +++ b/prisma/schema.prisma @@ -9077,10 +9077,10 @@ model InboundDocument { /// Why the most recent index attempt produced NO index — a closed string /// enum (`DocumentIndexOutcomeValue` in /// `src/lib/validations/inbound-documents.ts`): local-empty | - /// local-unsupported | decrypt-error | raster-failed | provider-error | - /// pdf-needs-anthropic | empty-transcription. Null after a successful - /// attempt (the `DocumentContentIndex` row itself is the success signal) - /// and before the first attempt. Lets the detail view tell "never tried" + /// local-unsupported | decrypt-error | resource-limit | raster-failed | + /// provider-error | pdf-needs-anthropic | empty-transcription. Null after a + /// successful attempt (the `DocumentContentIndex` row itself is the success + /// signal) and before the first attempt. Lets the detail view tell "never tried" /// from "tried and here is why it failed" instead of showing a bare /// not-indexed state for both. lastIndexOutcome String? @map("last_index_outcome") diff --git a/src/lib/documents/__tests__/index-document.test.ts b/src/lib/documents/__tests__/index-document.test.ts index 1a06e6d38..80a72e17a 100644 --- a/src/lib/documents/__tests__/index-document.test.ts +++ b/src/lib/documents/__tests__/index-document.test.ts @@ -290,6 +290,36 @@ describe("indexDocumentContent — local fallback path", () => { expect(outcome).toEqual({ indexed: false, reason: "local-empty" }); expect(upsertContentIndex).not.toHaveBeenCalled(); }); + + it("reports a PDF past the reader's limits as resource-limit, not as a file that would not open", async () => { + vi.mocked(resolveDocumentVisionProvider).mockResolvedValue({ + chain: [], + pick: null, + } as never); + vi.mocked(localExtractText).mockResolvedValue({ + ok: false, + reason: "resource-limit", + }); + + const outcome = await indexDocumentContent("user-1", "doc-1"); + expect(outcome).toEqual({ indexed: false, reason: "resource-limit" }); + expect(upsertContentIndex).not.toHaveBeenCalled(); + }); + + it("keeps resource-limit over a provider note: the provider never got further either", async () => { + vi.mocked(resolveDocumentVisionProvider).mockResolvedValue(PICK as never); + vi.mocked(prepareVisionInput).mockReturnValue({ + ok: false, + reason: "rasterFailed", + } as never); + vi.mocked(localExtractText).mockResolvedValue({ + ok: false, + reason: "resource-limit", + }); + + const outcome = await indexDocumentContent("user-1", "doc-1"); + expect(outcome).toEqual({ indexed: false, reason: "resource-limit" }); + }); }); describe("indexDocumentContent — the documentAi capability", () => { diff --git a/src/lib/documents/__tests__/local-extract-resource-limit.test.ts b/src/lib/documents/__tests__/local-extract-resource-limit.test.ts new file mode 100644 index 000000000..22663188f --- /dev/null +++ b/src/lib/documents/__tests__/local-extract-resource-limit.test.ts @@ -0,0 +1,53 @@ +import { describe, expect, it, vi } from "vitest"; + +/** + * A PDF that crosses the reader's memory or time bound is reported as such + * (`resource-limit`), not as a read error, which the index job records as a + * stored file that would not open. The bound itself is the isolated reader's + * (`../pdf-isolation.ts`); here it is the error it throws. + */ + +const { annotate, logCaught, readPdfText } = vi.hoisted(() => ({ + annotate: vi.fn(), + logCaught: vi.fn(), + readPdfText: vi.fn(), +})); +vi.mock("@/lib/logging/context", () => ({ annotate })); +vi.mock("@/lib/logging/signal", () => ({ logCaught })); +vi.mock("../pdfjs", async (importOriginal) => ({ + ...(await importOriginal()), + readPdfText, +})); + +import { PdfResourceLimitError } from "../pdfjs"; +import { extractPdfText, localExtractText } from "../local-extract"; + +describe("a PDF past the reader's limits", () => { + it.each(["memory", "time", "exit"] as const)( + "answers resource-limit for the %s bound, without the error log a broken file gets", + async (limit) => { + readPdfText.mockRejectedValueOnce(new PdfResourceLimitError(limit)); + annotate.mockClear(); + logCaught.mockClear(); + const bytes = Buffer.from("%PDF-1.7 large"); + expect(await extractPdfText(bytes)).toEqual({ + ok: false, + reason: "resource-limit", + }); + expect(logCaught).not.toHaveBeenCalled(); + expect(annotate).toHaveBeenCalledWith({ + action: { name: "documents.localIndex.resourceLimit" }, + meta: { limit, bytes: bytes.length }, + }); + }, + ); + + it("still answers error for anything else the reader throws", async () => { + const broken = new Error("Invalid PDF structure"); + broken.name = "InvalidPDFException"; + readPdfText.mockRejectedValueOnce(broken); + expect( + await localExtractText(Buffer.from("%PDF-1.7"), "application/pdf"), + ).toEqual({ ok: false, reason: "error" }); + }); +}); diff --git a/src/lib/documents/index-document.ts b/src/lib/documents/index-document.ts index c1fa4fa60..69ae2ff41 100644 --- a/src/lib/documents/index-document.ts +++ b/src/lib/documents/index-document.ts @@ -78,6 +78,7 @@ export type IndexFailureReason = | "local-empty" | "local-unsupported" | "decrypt-error" + | "resource-limit" | "raster-failed" | "provider-error" | "pdf-needs-anthropic" @@ -342,6 +343,12 @@ async function tryLocalIndex( if (result.reason === "unsupported") { return { indexed: false, reason: "local-unsupported" }; } + if (result.reason === "resource-limit") { + // Too large or too complex for the reader's bounds: said as such, and + // final like the thumbnail's refusal of the same bytes. Nothing retries + // it on its own; a person can still ask for a read. + return { indexed: false, reason: "resource-limit" }; + } if (result.reason === "error") { return { indexed: false, reason: "decrypt-error" }; } diff --git a/src/lib/documents/local-extract.ts b/src/lib/documents/local-extract.ts index 327f8a8b9..8469dc7c6 100644 --- a/src/lib/documents/local-extract.ts +++ b/src/lib/documents/local-extract.ts @@ -32,7 +32,7 @@ import { Buffer } from "node:buffer"; import { annotate } from "@/lib/logging/context"; import { logCaught } from "@/lib/logging/signal"; -import { readPdfText } from "./pdfjs"; +import { PdfResourceLimitError, readPdfText } from "./pdfjs"; /** * Minimum joined text length for a PDF to count as a real text-layer document. @@ -54,15 +54,23 @@ export type LocalExtractSource = "local-pdf" | "local-ocr"; * a provider vision pass is the only way to read it. * - `unsupported` → a MIME the local path cannot read without OCR (images) or * at all; leave un-indexed locally. + * - `resource-limit` → the PDF crossed the reader's memory or time bound + * (`PdfResourceLimitError`): too large or too complex to + * read here. The same bytes cross it again, so it is a + * final answer, as it is for thumbnails. * - `error` → the file was malformed / unreadable; treat as un-indexed. */ export type LocalExtractResult = | { ok: true; text: string; source: LocalExtractSource } - | { ok: false; reason: "empty" | "unsupported" | "error" }; + | { + ok: false; + reason: "empty" | "unsupported" | "resource-limit" | "error"; + }; /** * Extract a text-layer PDF's embedded text server-side. Returns `empty` for a - * scanned/no-text-layer PDF and `error` for a malformed/encrypted one — never + * scanned/no-text-layer PDF, `resource-limit` for one past the reader's memory + * or time bound, and `error` for a malformed/encrypted one — never * throws, so a bad document can never abort the caller (the upload/backfill job * just leaves it un-indexed). An `error` leaves a `warn` line naming what * pdf.js threw, because the outcome alone does not say why a document stayed @@ -76,6 +84,15 @@ export async function extractPdfText( // Bound the page walk so a pathological many-page PDF costs no more. text = (await readPdfText(buffer, LOCAL_MAX_PAGES)).trim(); } catch (err) { + if (err instanceof PdfResourceLimitError) { + // Not a fault of the file's bytes being unreadable: they are readable, + // only not within the bounds this server gives one document. + annotate({ + action: { name: "documents.localIndex.resourceLimit" }, + meta: { limit: err.limit, bytes: buffer.length }, + }); + return { ok: false, reason: "resource-limit" }; + } const errorType = err instanceof Error ? err.name || "Error" : typeof err; annotate({ action: { name: "documents.localIndex.extractFailed" }, diff --git a/src/lib/openapi/routes/documents.ts b/src/lib/openapi/routes/documents.ts index 807c7e7f1..43f41cd77 100644 --- a/src/lib/openapi/routes/documents.ts +++ b/src/lib/openapi/routes/documents.ts @@ -300,6 +300,7 @@ const inboundDocument = z "local-empty", "local-unsupported", "decrypt-error", + "resource-limit", "raster-failed", "provider-error", "pdf-needs-anthropic", diff --git a/src/lib/validations/inbound-documents.ts b/src/lib/validations/inbound-documents.ts index 7536865bd..a83a786d1 100644 --- a/src/lib/validations/inbound-documents.ts +++ b/src/lib/validations/inbound-documents.ts @@ -115,6 +115,7 @@ export const DOCUMENT_INDEX_OUTCOMES = [ "local-empty", "local-unsupported", "decrypt-error", + "resource-limit", "raster-failed", "provider-error", "pdf-needs-anthropic",