diff --git a/docs/api/openapi.yaml b/docs/api/openapi.yaml index 7f5523d68..02f8a3095 100644 --- a/docs/api/openapi.yaml +++ b/docs/api/openapi.yaml @@ -79798,6 +79798,7 @@ components: - local-empty - local-unsupported - decrypt-error + - resource-limit - raster-failed - provider-error - pdf-needs-anthropic @@ -80370,6 +80371,7 @@ components: - local-empty - local-unsupported - decrypt-error + - resource-limit - raster-failed - provider-error - pdf-needs-anthropic diff --git a/messages/de.json b/messages/de.json index bb2624eeb..96a78eff5 100644 --- a/messages/de.json +++ b/messages/de.json @@ -11877,7 +11877,8 @@ "raster-failed": "Die PDF-Seiten konnten zum Lesen nicht gerendert werden. Gescannte Fax-Kompressionsformate sind eine bekannte Ursache.", "provider-error": "Der KI-Anbieter konnte dieses Dokument beim letzten Versuch nicht lesen.", "pdf-needs-anthropic": "Der eingerichtete KI-Anbieter kann keine PDF-Dateien lesen. Ein Claude-Anbieter oder lokale OCR kann es.", - "empty-transcription": "Der KI-Anbieter hat für dieses Dokument keinen Text geliefert." + "empty-transcription": "Der KI-Anbieter hat für dieses Dokument keinen Text geliefert.", + "resource-limit": "Dieses PDF ist zu groß oder zu komplex, um es hier zu lesen. Die Datei selbst ist in Ordnung und lässt sich weiterhin öffnen." } }, "loadError": "Dieses Dokument konnte nicht geladen werden.", diff --git a/messages/en.json b/messages/en.json index f99c0d281..bb4cf34c4 100644 --- a/messages/en.json +++ b/messages/en.json @@ -11877,7 +11877,8 @@ "raster-failed": "The PDF pages couldn't be rendered for reading. Scanned fax compression formats are a known cause.", "provider-error": "The AI provider couldn't read this document on the last try.", "pdf-needs-anthropic": "The configured AI provider can't read PDF files. A Claude provider or local OCR can.", - "empty-transcription": "The AI provider returned no text for this document." + "empty-transcription": "The AI provider returned no text for this document.", + "resource-limit": "This PDF is too large or complex to read here. The file itself is fine and still opens." } }, "loadError": "Couldn't load this document.", diff --git a/messages/es.json b/messages/es.json index 37bf14e28..5c9056ae2 100644 --- a/messages/es.json +++ b/messages/es.json @@ -11877,7 +11877,8 @@ "raster-failed": "Las páginas del PDF no se pudieron procesar para su lectura. Los formatos de compresión de fax escaneados son una causa conocida.", "provider-error": "El proveedor de IA no pudo leer este documento en el último intento.", "pdf-needs-anthropic": "El proveedor de IA configurado no puede leer archivos PDF. Un proveedor Claude o el OCR local sí pueden.", - "empty-transcription": "El proveedor de IA no devolvió texto para este documento." + "empty-transcription": "El proveedor de IA no devolvió texto para este documento.", + "resource-limit": "Este PDF es demasiado grande o complejo para leerlo aquí. El archivo en sí está bien y se sigue pudiendo abrir." } }, "loadError": "No se pudo cargar este documento.", diff --git a/messages/fr.json b/messages/fr.json index 9afb8acda..ddc754baf 100644 --- a/messages/fr.json +++ b/messages/fr.json @@ -11877,7 +11877,8 @@ "raster-failed": "Les pages du PDF n’ont pas pu être rendues pour la lecture. Les formats de compression de fax numérisés en sont une cause connue.", "provider-error": "Le fournisseur d’IA n’a pas pu lire ce document lors de la dernière tentative.", "pdf-needs-anthropic": "Le fournisseur d’IA configuré ne peut pas lire les fichiers PDF. Un fournisseur Claude ou l’OCR local le peut.", - "empty-transcription": "Le fournisseur d’IA n’a renvoyé aucun texte pour ce document." + "empty-transcription": "Le fournisseur d’IA n’a renvoyé aucun texte pour ce document.", + "resource-limit": "Ce PDF est trop volumineux ou trop complexe pour être lu ici. Le fichier lui-même est intact et s'ouvre toujours." } }, "loadError": "Impossible de charger ce document.", diff --git a/messages/it.json b/messages/it.json index a67d5d50e..fe6450c8d 100644 --- a/messages/it.json +++ b/messages/it.json @@ -11877,7 +11877,8 @@ "raster-failed": "Le pagine del PDF non possono essere elaborate per la lettura. I formati di compressione fax digitalizzati sono una causa nota.", "provider-error": "Il provider IA non è riuscito a leggere questo documento all’ultimo tentativo.", "pdf-needs-anthropic": "Il provider IA configurato non può leggere i file PDF. Un provider Claude o l’OCR locale sì.", - "empty-transcription": "Il provider IA non ha restituito testo per questo documento." + "empty-transcription": "Il provider IA non ha restituito testo per questo documento.", + "resource-limit": "Questo PDF è troppo grande o complesso per essere letto qui. Il file in sé è a posto e si apre ancora." } }, "loadError": "Impossibile caricare questo documento.", diff --git a/messages/ko.json b/messages/ko.json index 1caf6faf9..2ef2164d3 100644 --- a/messages/ko.json +++ b/messages/ko.json @@ -11877,7 +11877,8 @@ "raster-failed": "PDF 페이지를 읽기용으로 렌더링하지 못했어요. 스캔 팩스 압축 형식이 흔한 원인이에요.", "provider-error": "지난번에 AI 제공자가 이 문서를 읽지 못했어요.", "pdf-needs-anthropic": "설정된 AI 제공자는 PDF 파일을 읽을 수 없어요. Claude 제공자나 로컬 OCR로는 읽을 수 있어요.", - "empty-transcription": "AI 제공자가 이 문서에서 텍스트를 찾지 못했어요." + "empty-transcription": "AI 제공자가 이 문서에서 텍스트를 찾지 못했어요.", + "resource-limit": "이 PDF는 너무 크거나 복잡해서 여기서 읽을 수 없습니다. 파일 자체는 정상이며 계속 열 수 있습니다." } }, "loadError": "이 문서를 불러오지 못했어요.", diff --git a/messages/nl.json b/messages/nl.json index 01901ab56..9bd5560ec 100644 --- a/messages/nl.json +++ b/messages/nl.json @@ -11877,7 +11877,8 @@ "raster-failed": "De PDF-pagina's konden niet worden weergegeven om te lezen. Compressieformaten van gescande faxen zijn een bekende oorzaak.", "provider-error": "De AI-provider kon dit document bij de laatste poging niet lezen.", "pdf-needs-anthropic": "De ingestelde AI-provider kan geen PDF-bestanden lezen. Een Claude-provider of lokale OCR kan dat wel.", - "empty-transcription": "De AI-provider gaf geen tekst terug voor dit document." + "empty-transcription": "De AI-provider gaf geen tekst terug voor dit document.", + "resource-limit": "Deze pdf is te groot of te complex om hier te lezen. Het bestand zelf is in orde en opent nog steeds." } }, "loadError": "Dit document kon niet worden geladen.", diff --git a/messages/pl.json b/messages/pl.json index c498fc660..2a1673bcd 100644 --- a/messages/pl.json +++ b/messages/pl.json @@ -11877,7 +11877,8 @@ "raster-failed": "Nie udało się przetworzyć stron PDF do odczytu. Znaną przyczyną są skany w faksowych formatach kompresji.", "provider-error": "Dostawca AI nie zdołał odczytać tego dokumentu przy ostatniej próbie.", "pdf-needs-anthropic": "Skonfigurowany dostawca AI nie odczytuje plików PDF. Poradzi sobie z tym dostawca Claude lub lokalny OCR.", - "empty-transcription": "Dostawca AI nie zwrócił tekstu dla tego dokumentu." + "empty-transcription": "Dostawca AI nie zwrócił tekstu dla tego dokumentu.", + "resource-limit": "Ten PDF jest zbyt duży lub zbyt złożony, żeby odczytać go tutaj. Sam plik jest w porządku i nadal się otwiera." } }, "loadError": "Nie udało się wczytać tego dokumentu.", diff --git a/messages/pt-BR.json b/messages/pt-BR.json index ab7eaf3ca..cc011e70a 100644 --- a/messages/pt-BR.json +++ b/messages/pt-BR.json @@ -11877,7 +11877,8 @@ "raster-failed": "As páginas do PDF não puderam ser renderizadas para leitura. Formatos de compressão de fax digitalizado são uma causa conhecida.", "provider-error": "O provedor de IA não conseguiu ler este documento na última tentativa.", "pdf-needs-anthropic": "O provedor de IA configurado não consegue ler arquivos PDF. Um provedor Claude ou OCR local consegue.", - "empty-transcription": "O provedor de IA não retornou texto para este documento." + "empty-transcription": "O provedor de IA não retornou texto para este documento.", + "resource-limit": "Este PDF é grande ou complexo demais para ser lido aqui. O arquivo em si está bem e continua abrindo." } }, "loadError": "Não foi possível carregar este documento.", diff --git a/prisma/schema.prisma b/prisma/schema.prisma index 67cfbeb4e..2b5fd6b2c 100644 --- a/prisma/schema.prisma +++ b/prisma/schema.prisma @@ -9077,10 +9077,10 @@ model InboundDocument { /// Why the most recent index attempt produced NO index — a closed string /// enum (`DocumentIndexOutcomeValue` in /// `src/lib/validations/inbound-documents.ts`): local-empty | - /// local-unsupported | decrypt-error | raster-failed | provider-error | - /// pdf-needs-anthropic | empty-transcription. Null after a successful - /// attempt (the `DocumentContentIndex` row itself is the success signal) - /// and before the first attempt. Lets the detail view tell "never tried" + /// local-unsupported | decrypt-error | resource-limit | raster-failed | + /// provider-error | pdf-needs-anthropic | empty-transcription. Null after a + /// successful attempt (the `DocumentContentIndex` row itself is the success + /// signal) and before the first attempt. Lets the detail view tell "never tried" /// from "tried and here is why it failed" instead of showing a bare /// not-indexed state for both. lastIndexOutcome String? @map("last_index_outcome") diff --git a/src/lib/documents/__tests__/index-document.test.ts b/src/lib/documents/__tests__/index-document.test.ts index 1a06e6d38..80a72e17a 100644 --- a/src/lib/documents/__tests__/index-document.test.ts +++ b/src/lib/documents/__tests__/index-document.test.ts @@ -290,6 +290,36 @@ describe("indexDocumentContent — local fallback path", () => { expect(outcome).toEqual({ indexed: false, reason: "local-empty" }); expect(upsertContentIndex).not.toHaveBeenCalled(); }); + + it("reports a PDF past the reader's limits as resource-limit, not as a file that would not open", async () => { + vi.mocked(resolveDocumentVisionProvider).mockResolvedValue({ + chain: [], + pick: null, + } as never); + vi.mocked(localExtractText).mockResolvedValue({ + ok: false, + reason: "resource-limit", + }); + + const outcome = await indexDocumentContent("user-1", "doc-1"); + expect(outcome).toEqual({ indexed: false, reason: "resource-limit" }); + expect(upsertContentIndex).not.toHaveBeenCalled(); + }); + + it("keeps resource-limit over a provider note: the provider never got further either", async () => { + vi.mocked(resolveDocumentVisionProvider).mockResolvedValue(PICK as never); + vi.mocked(prepareVisionInput).mockReturnValue({ + ok: false, + reason: "rasterFailed", + } as never); + vi.mocked(localExtractText).mockResolvedValue({ + ok: false, + reason: "resource-limit", + }); + + const outcome = await indexDocumentContent("user-1", "doc-1"); + expect(outcome).toEqual({ indexed: false, reason: "resource-limit" }); + }); }); describe("indexDocumentContent — the documentAi capability", () => { diff --git a/src/lib/documents/__tests__/local-extract-resource-limit.test.ts b/src/lib/documents/__tests__/local-extract-resource-limit.test.ts new file mode 100644 index 000000000..22663188f --- /dev/null +++ b/src/lib/documents/__tests__/local-extract-resource-limit.test.ts @@ -0,0 +1,53 @@ +import { describe, expect, it, vi } from "vitest"; + +/** + * A PDF that crosses the reader's memory or time bound is reported as such + * (`resource-limit`), not as a read error, which the index job records as a + * stored file that would not open. The bound itself is the isolated reader's + * (`../pdf-isolation.ts`); here it is the error it throws. + */ + +const { annotate, logCaught, readPdfText } = vi.hoisted(() => ({ + annotate: vi.fn(), + logCaught: vi.fn(), + readPdfText: vi.fn(), +})); +vi.mock("@/lib/logging/context", () => ({ annotate })); +vi.mock("@/lib/logging/signal", () => ({ logCaught })); +vi.mock("../pdfjs", async (importOriginal) => ({ + ...(await importOriginal()), + readPdfText, +})); + +import { PdfResourceLimitError } from "../pdfjs"; +import { extractPdfText, localExtractText } from "../local-extract"; + +describe("a PDF past the reader's limits", () => { + it.each(["memory", "time", "exit"] as const)( + "answers resource-limit for the %s bound, without the error log a broken file gets", + async (limit) => { + readPdfText.mockRejectedValueOnce(new PdfResourceLimitError(limit)); + annotate.mockClear(); + logCaught.mockClear(); + const bytes = Buffer.from("%PDF-1.7 large"); + expect(await extractPdfText(bytes)).toEqual({ + ok: false, + reason: "resource-limit", + }); + expect(logCaught).not.toHaveBeenCalled(); + expect(annotate).toHaveBeenCalledWith({ + action: { name: "documents.localIndex.resourceLimit" }, + meta: { limit, bytes: bytes.length }, + }); + }, + ); + + it("still answers error for anything else the reader throws", async () => { + const broken = new Error("Invalid PDF structure"); + broken.name = "InvalidPDFException"; + readPdfText.mockRejectedValueOnce(broken); + expect( + await localExtractText(Buffer.from("%PDF-1.7"), "application/pdf"), + ).toEqual({ ok: false, reason: "error" }); + }); +}); diff --git a/src/lib/documents/index-document.ts b/src/lib/documents/index-document.ts index c1fa4fa60..69ae2ff41 100644 --- a/src/lib/documents/index-document.ts +++ b/src/lib/documents/index-document.ts @@ -78,6 +78,7 @@ export type IndexFailureReason = | "local-empty" | "local-unsupported" | "decrypt-error" + | "resource-limit" | "raster-failed" | "provider-error" | "pdf-needs-anthropic" @@ -342,6 +343,12 @@ async function tryLocalIndex( if (result.reason === "unsupported") { return { indexed: false, reason: "local-unsupported" }; } + if (result.reason === "resource-limit") { + // Too large or too complex for the reader's bounds: said as such, and + // final like the thumbnail's refusal of the same bytes. Nothing retries + // it on its own; a person can still ask for a read. + return { indexed: false, reason: "resource-limit" }; + } if (result.reason === "error") { return { indexed: false, reason: "decrypt-error" }; } diff --git a/src/lib/documents/local-extract.ts b/src/lib/documents/local-extract.ts index 327f8a8b9..8469dc7c6 100644 --- a/src/lib/documents/local-extract.ts +++ b/src/lib/documents/local-extract.ts @@ -32,7 +32,7 @@ import { Buffer } from "node:buffer"; import { annotate } from "@/lib/logging/context"; import { logCaught } from "@/lib/logging/signal"; -import { readPdfText } from "./pdfjs"; +import { PdfResourceLimitError, readPdfText } from "./pdfjs"; /** * Minimum joined text length for a PDF to count as a real text-layer document. @@ -54,15 +54,23 @@ export type LocalExtractSource = "local-pdf" | "local-ocr"; * a provider vision pass is the only way to read it. * - `unsupported` → a MIME the local path cannot read without OCR (images) or * at all; leave un-indexed locally. + * - `resource-limit` → the PDF crossed the reader's memory or time bound + * (`PdfResourceLimitError`): too large or too complex to + * read here. The same bytes cross it again, so it is a + * final answer, as it is for thumbnails. * - `error` → the file was malformed / unreadable; treat as un-indexed. */ export type LocalExtractResult = | { ok: true; text: string; source: LocalExtractSource } - | { ok: false; reason: "empty" | "unsupported" | "error" }; + | { + ok: false; + reason: "empty" | "unsupported" | "resource-limit" | "error"; + }; /** * Extract a text-layer PDF's embedded text server-side. Returns `empty` for a - * scanned/no-text-layer PDF and `error` for a malformed/encrypted one — never + * scanned/no-text-layer PDF, `resource-limit` for one past the reader's memory + * or time bound, and `error` for a malformed/encrypted one — never * throws, so a bad document can never abort the caller (the upload/backfill job * just leaves it un-indexed). An `error` leaves a `warn` line naming what * pdf.js threw, because the outcome alone does not say why a document stayed @@ -76,6 +84,15 @@ export async function extractPdfText( // Bound the page walk so a pathological many-page PDF costs no more. text = (await readPdfText(buffer, LOCAL_MAX_PAGES)).trim(); } catch (err) { + if (err instanceof PdfResourceLimitError) { + // Not a fault of the file's bytes being unreadable: they are readable, + // only not within the bounds this server gives one document. + annotate({ + action: { name: "documents.localIndex.resourceLimit" }, + meta: { limit: err.limit, bytes: buffer.length }, + }); + return { ok: false, reason: "resource-limit" }; + } const errorType = err instanceof Error ? err.name || "Error" : typeof err; annotate({ action: { name: "documents.localIndex.extractFailed" }, diff --git a/src/lib/openapi/routes/documents.ts b/src/lib/openapi/routes/documents.ts index 807c7e7f1..43f41cd77 100644 --- a/src/lib/openapi/routes/documents.ts +++ b/src/lib/openapi/routes/documents.ts @@ -300,6 +300,7 @@ const inboundDocument = z "local-empty", "local-unsupported", "decrypt-error", + "resource-limit", "raster-failed", "provider-error", "pdf-needs-anthropic", diff --git a/src/lib/validations/inbound-documents.ts b/src/lib/validations/inbound-documents.ts index 7536865bd..a83a786d1 100644 --- a/src/lib/validations/inbound-documents.ts +++ b/src/lib/validations/inbound-documents.ts @@ -115,6 +115,7 @@ export const DOCUMENT_INDEX_OUTCOMES = [ "local-empty", "local-unsupported", "decrypt-error", + "resource-limit", "raster-failed", "provider-error", "pdf-needs-anthropic",