fix(documents): say a PDF past the reader's limits is too large to read

A text read that crossed the isolated reader's memory or time bound came
back as a plain read error, which the index job records as decrypt-error,
so the document said its stored file could not be opened. The local
extractor now answers resource-limit for PdfResourceLimitError, the index
records it under that name, and the detail view says the PDF is too large
or complex to read here, in every locale. Like the thumbnail's refusal of
the same bytes it is final: nothing retries it on its own.
This commit is contained in:
Marc-André Bombeck committed 2026-10-25 16:00:35 +01:00
1 parent 66e92f36fe
commit 8471e0edc9
17 files changed
+136 -16

No files matched your search

@@ -290,6 +290,36 @@ describe("indexDocumentContent — local fallback path", () => {
expect(outcome).toEqual({ indexed: false, reason: "local-empty" });
expect(upsertContentIndex).not.toHaveBeenCalled();
});
it("reports a PDF past the reader's limits as resource-limit, not as a file that would not open", async () => {
vi.mocked(resolveDocumentVisionProvider).mockResolvedValue({
chain: [],
pick: null,
} as never);
vi.mocked(localExtractText).mockResolvedValue({
ok: false,
reason: "resource-limit",
});
const outcome = await indexDocumentContent("user-1", "doc-1");
expect(outcome).toEqual({ indexed: false, reason: "resource-limit" });
expect(upsertContentIndex).not.toHaveBeenCalled();
});
it("keeps resource-limit over a provider note: the provider never got further either", async () => {
vi.mocked(resolveDocumentVisionProvider).mockResolvedValue(PICK as never);
vi.mocked(prepareVisionInput).mockReturnValue({
ok: false,
reason: "rasterFailed",
} as never);
vi.mocked(localExtractText).mockResolvedValue({
ok: false,
reason: "resource-limit",
});
const outcome = await indexDocumentContent("user-1", "doc-1");
expect(outcome).toEqual({ indexed: false, reason: "resource-limit" });
});
});
describe("indexDocumentContent — the documentAi capability", () => {
@@ -0,0 +1,53 @@
import { describe, expect, it, vi } from "vitest";
/**
* A PDF that crosses the reader's memory or time bound is reported as such
* (`resource-limit`), not as a read error, which the index job records as a
* stored file that would not open. The bound itself is the isolated reader's
* (`../pdf-isolation.ts`); here it is the error it throws.
*/
const { annotate, logCaught, readPdfText } = vi.hoisted(() => ({
annotate: vi.fn(),
logCaught: vi.fn(),
readPdfText: vi.fn(),
}));
vi.mock("@/lib/logging/context", () => ({ annotate }));
vi.mock("@/lib/logging/signal", () => ({ logCaught }));
vi.mock("../pdfjs", async (importOriginal) => ({
...(await importOriginal<typeof import("../pdfjs")>()),
readPdfText,
}));
import { PdfResourceLimitError } from "../pdfjs";
import { extractPdfText, localExtractText } from "../local-extract";
describe("a PDF past the reader's limits", () => {
it.each(["memory", "time", "exit"] as const)(
"answers resource-limit for the %s bound, without the error log a broken file gets",
async (limit) => {
readPdfText.mockRejectedValueOnce(new PdfResourceLimitError(limit));
annotate.mockClear();
logCaught.mockClear();
const bytes = Buffer.from("%PDF-1.7 large");
expect(await extractPdfText(bytes)).toEqual({
ok: false,
reason: "resource-limit",
});
expect(logCaught).not.toHaveBeenCalled();
expect(annotate).toHaveBeenCalledWith({
action: { name: "documents.localIndex.resourceLimit" },
meta: { limit, bytes: bytes.length },
});
},
);
it("still answers error for anything else the reader throws", async () => {
const broken = new Error("Invalid PDF structure");
broken.name = "InvalidPDFException";
readPdfText.mockRejectedValueOnce(broken);
expect(
await localExtractText(Buffer.from("%PDF-1.7"), "application/pdf"),
).toEqual({ ok: false, reason: "error" });
});
});
+7
View File
@@ -78,6 +78,7 @@ export type IndexFailureReason =
| "local-empty"
| "local-unsupported"
| "decrypt-error"
| "resource-limit"
| "raster-failed"
| "provider-error"
| "pdf-needs-anthropic"
@@ -342,6 +343,12 @@ async function tryLocalIndex(
if (result.reason === "unsupported") {
return { indexed: false, reason: "local-unsupported" };
}
if (result.reason === "resource-limit") {
// Too large or too complex for the reader's bounds: said as such, and
// final like the thumbnail's refusal of the same bytes. Nothing retries
// it on its own; a person can still ask for a read.
return { indexed: false, reason: "resource-limit" };
}
if (result.reason === "error") {
return { indexed: false, reason: "decrypt-error" };
}
+20 -3
View File
@@ -32,7 +32,7 @@ import { Buffer } from "node:buffer";
import { annotate } from "@/lib/logging/context";
import { logCaught } from "@/lib/logging/signal";
import { readPdfText } from "./pdfjs";
import { PdfResourceLimitError, readPdfText } from "./pdfjs";
/**
* Minimum joined text length for a PDF to count as a real text-layer document.
@@ -54,15 +54,23 @@ export type LocalExtractSource = "local-pdf" | "local-ocr";
* a provider vision pass is the only way to read it.
* - `unsupported` → a MIME the local path cannot read without OCR (images) or
* at all; leave un-indexed locally.
* - `resource-limit` → the PDF crossed the reader's memory or time bound
* (`PdfResourceLimitError`): too large or too complex to
* read here. The same bytes cross it again, so it is a
* final answer, as it is for thumbnails.
* - `error` → the file was malformed / unreadable; treat as un-indexed.
*/
export type LocalExtractResult =
| { ok: true; text: string; source: LocalExtractSource }
| { ok: false; reason: "empty" | "unsupported" | "error" };
| {
ok: false;
reason: "empty" | "unsupported" | "resource-limit" | "error";
};
/**
* Extract a text-layer PDF's embedded text server-side. Returns `empty` for a
* scanned/no-text-layer PDF and `error` for a malformed/encrypted one — never
* scanned/no-text-layer PDF, `resource-limit` for one past the reader's memory
* or time bound, and `error` for a malformed/encrypted one — never
* throws, so a bad document can never abort the caller (the upload/backfill job
* just leaves it un-indexed). An `error` leaves a `warn` line naming what
* pdf.js threw, because the outcome alone does not say why a document stayed
@@ -76,6 +84,15 @@ export async function extractPdfText(
// Bound the page walk so a pathological many-page PDF costs no more.
text = (await readPdfText(buffer, LOCAL_MAX_PAGES)).trim();
} catch (err) {
if (err instanceof PdfResourceLimitError) {
// Not a fault of the file's bytes being unreadable: they are readable,
// only not within the bounds this server gives one document.
annotate({
action: { name: "documents.localIndex.resourceLimit" },
meta: { limit: err.limit, bytes: buffer.length },
});
return { ok: false, reason: "resource-limit" };
}
const errorType = err instanceof Error ? err.name || "Error" : typeof err;
annotate({
action: { name: "documents.localIndex.extractFailed" },
+1
View File
@@ -300,6 +300,7 @@ const inboundDocument = z
"local-empty",
"local-unsupported",
"decrypt-error",
"resource-limit",
"raster-failed",
"provider-error",
"pdf-needs-anthropic",
+1
View File
@@ -115,6 +115,7 @@ export const DOCUMENT_INDEX_OUTCOMES = [
"local-empty",
"local-unsupported",
"decrypt-error",
"resource-limit",
"raster-failed",
"provider-error",
"pdf-needs-anthropic",