mirror of
https://github.com/MBombeck/HealthLog.git
synced 2026-11-09 04:50:59 +00:00
fix(documents): say a PDF past the reader's limits is too large to read
A text read that crossed the isolated reader's memory or time bound came back as a plain read error, which the index job records as decrypt-error, so the document said its stored file could not be opened. The local extractor now answers resource-limit for PdfResourceLimitError, the index records it under that name, and the detail view says the PDF is too large or complex to read here, in every locale. Like the thumbnail's refusal of the same bytes it is final: nothing retries it on its own.
This commit is contained in:
1 parent
66e92f36fe
commit
8471e0edc9
17 files changed
+136
-16
No files matched your search
@@ -79798,6 +79798,7 @@ components:
|
||||
- local-empty
|
||||
- local-unsupported
|
||||
- decrypt-error
|
||||
- resource-limit
|
||||
- raster-failed
|
||||
- provider-error
|
||||
- pdf-needs-anthropic
|
||||
@@ -80370,6 +80371,7 @@ components:
|
||||
- local-empty
|
||||
- local-unsupported
|
||||
- decrypt-error
|
||||
- resource-limit
|
||||
- raster-failed
|
||||
- provider-error
|
||||
- pdf-needs-anthropic
|
||||
|
||||
+2
-1
@@ -11877,7 +11877,8 @@
|
||||
"raster-failed": "Die PDF-Seiten konnten zum Lesen nicht gerendert werden. Gescannte Fax-Kompressionsformate sind eine bekannte Ursache.",
|
||||
"provider-error": "Der KI-Anbieter konnte dieses Dokument beim letzten Versuch nicht lesen.",
|
||||
"pdf-needs-anthropic": "Der eingerichtete KI-Anbieter kann keine PDF-Dateien lesen. Ein Claude-Anbieter oder lokale OCR kann es.",
|
||||
"empty-transcription": "Der KI-Anbieter hat für dieses Dokument keinen Text geliefert."
|
||||
"empty-transcription": "Der KI-Anbieter hat für dieses Dokument keinen Text geliefert.",
|
||||
"resource-limit": "Dieses PDF ist zu groß oder zu komplex, um es hier zu lesen. Die Datei selbst ist in Ordnung und lässt sich weiterhin öffnen."
|
||||
}
|
||||
},
|
||||
"loadError": "Dieses Dokument konnte nicht geladen werden.",
|
||||
|
||||
+2
-1
@@ -11877,7 +11877,8 @@
|
||||
"raster-failed": "The PDF pages couldn't be rendered for reading. Scanned fax compression formats are a known cause.",
|
||||
"provider-error": "The AI provider couldn't read this document on the last try.",
|
||||
"pdf-needs-anthropic": "The configured AI provider can't read PDF files. A Claude provider or local OCR can.",
|
||||
"empty-transcription": "The AI provider returned no text for this document."
|
||||
"empty-transcription": "The AI provider returned no text for this document.",
|
||||
"resource-limit": "This PDF is too large or complex to read here. The file itself is fine and still opens."
|
||||
}
|
||||
},
|
||||
"loadError": "Couldn't load this document.",
|
||||
|
||||
+2
-1
@@ -11877,7 +11877,8 @@
|
||||
"raster-failed": "Las páginas del PDF no se pudieron procesar para su lectura. Los formatos de compresión de fax escaneados son una causa conocida.",
|
||||
"provider-error": "El proveedor de IA no pudo leer este documento en el último intento.",
|
||||
"pdf-needs-anthropic": "El proveedor de IA configurado no puede leer archivos PDF. Un proveedor Claude o el OCR local sí pueden.",
|
||||
"empty-transcription": "El proveedor de IA no devolvió texto para este documento."
|
||||
"empty-transcription": "El proveedor de IA no devolvió texto para este documento.",
|
||||
"resource-limit": "Este PDF es demasiado grande o complejo para leerlo aquí. El archivo en sí está bien y se sigue pudiendo abrir."
|
||||
}
|
||||
},
|
||||
"loadError": "No se pudo cargar este documento.",
|
||||
|
||||
+2
-1
@@ -11877,7 +11877,8 @@
|
||||
"raster-failed": "Les pages du PDF n’ont pas pu être rendues pour la lecture. Les formats de compression de fax numérisés en sont une cause connue.",
|
||||
"provider-error": "Le fournisseur d’IA n’a pas pu lire ce document lors de la dernière tentative.",
|
||||
"pdf-needs-anthropic": "Le fournisseur d’IA configuré ne peut pas lire les fichiers PDF. Un fournisseur Claude ou l’OCR local le peut.",
|
||||
"empty-transcription": "Le fournisseur d’IA n’a renvoyé aucun texte pour ce document."
|
||||
"empty-transcription": "Le fournisseur d’IA n’a renvoyé aucun texte pour ce document.",
|
||||
"resource-limit": "Ce PDF est trop volumineux ou trop complexe pour être lu ici. Le fichier lui-même est intact et s'ouvre toujours."
|
||||
}
|
||||
},
|
||||
"loadError": "Impossible de charger ce document.",
|
||||
|
||||
+2
-1
@@ -11877,7 +11877,8 @@
|
||||
"raster-failed": "Le pagine del PDF non possono essere elaborate per la lettura. I formati di compressione fax digitalizzati sono una causa nota.",
|
||||
"provider-error": "Il provider IA non è riuscito a leggere questo documento all’ultimo tentativo.",
|
||||
"pdf-needs-anthropic": "Il provider IA configurato non può leggere i file PDF. Un provider Claude o l’OCR locale sì.",
|
||||
"empty-transcription": "Il provider IA non ha restituito testo per questo documento."
|
||||
"empty-transcription": "Il provider IA non ha restituito testo per questo documento.",
|
||||
"resource-limit": "Questo PDF è troppo grande o complesso per essere letto qui. Il file in sé è a posto e si apre ancora."
|
||||
}
|
||||
},
|
||||
"loadError": "Impossibile caricare questo documento.",
|
||||
|
||||
+2
-1
@@ -11877,7 +11877,8 @@
|
||||
"raster-failed": "PDF 페이지를 읽기용으로 렌더링하지 못했어요. 스캔 팩스 압축 형식이 흔한 원인이에요.",
|
||||
"provider-error": "지난번에 AI 제공자가 이 문서를 읽지 못했어요.",
|
||||
"pdf-needs-anthropic": "설정된 AI 제공자는 PDF 파일을 읽을 수 없어요. Claude 제공자나 로컬 OCR로는 읽을 수 있어요.",
|
||||
"empty-transcription": "AI 제공자가 이 문서에서 텍스트를 찾지 못했어요."
|
||||
"empty-transcription": "AI 제공자가 이 문서에서 텍스트를 찾지 못했어요.",
|
||||
"resource-limit": "이 PDF는 너무 크거나 복잡해서 여기서 읽을 수 없습니다. 파일 자체는 정상이며 계속 열 수 있습니다."
|
||||
}
|
||||
},
|
||||
"loadError": "이 문서를 불러오지 못했어요.",
|
||||
|
||||
+2
-1
@@ -11877,7 +11877,8 @@
|
||||
"raster-failed": "De PDF-pagina's konden niet worden weergegeven om te lezen. Compressieformaten van gescande faxen zijn een bekende oorzaak.",
|
||||
"provider-error": "De AI-provider kon dit document bij de laatste poging niet lezen.",
|
||||
"pdf-needs-anthropic": "De ingestelde AI-provider kan geen PDF-bestanden lezen. Een Claude-provider of lokale OCR kan dat wel.",
|
||||
"empty-transcription": "De AI-provider gaf geen tekst terug voor dit document."
|
||||
"empty-transcription": "De AI-provider gaf geen tekst terug voor dit document.",
|
||||
"resource-limit": "Deze pdf is te groot of te complex om hier te lezen. Het bestand zelf is in orde en opent nog steeds."
|
||||
}
|
||||
},
|
||||
"loadError": "Dit document kon niet worden geladen.",
|
||||
|
||||
+2
-1
@@ -11877,7 +11877,8 @@
|
||||
"raster-failed": "Nie udało się przetworzyć stron PDF do odczytu. Znaną przyczyną są skany w faksowych formatach kompresji.",
|
||||
"provider-error": "Dostawca AI nie zdołał odczytać tego dokumentu przy ostatniej próbie.",
|
||||
"pdf-needs-anthropic": "Skonfigurowany dostawca AI nie odczytuje plików PDF. Poradzi sobie z tym dostawca Claude lub lokalny OCR.",
|
||||
"empty-transcription": "Dostawca AI nie zwrócił tekstu dla tego dokumentu."
|
||||
"empty-transcription": "Dostawca AI nie zwrócił tekstu dla tego dokumentu.",
|
||||
"resource-limit": "Ten PDF jest zbyt duży lub zbyt złożony, żeby odczytać go tutaj. Sam plik jest w porządku i nadal się otwiera."
|
||||
}
|
||||
},
|
||||
"loadError": "Nie udało się wczytać tego dokumentu.",
|
||||
|
||||
+2
-1
@@ -11877,7 +11877,8 @@
|
||||
"raster-failed": "As páginas do PDF não puderam ser renderizadas para leitura. Formatos de compressão de fax digitalizado são uma causa conhecida.",
|
||||
"provider-error": "O provedor de IA não conseguiu ler este documento na última tentativa.",
|
||||
"pdf-needs-anthropic": "O provedor de IA configurado não consegue ler arquivos PDF. Um provedor Claude ou OCR local consegue.",
|
||||
"empty-transcription": "O provedor de IA não retornou texto para este documento."
|
||||
"empty-transcription": "O provedor de IA não retornou texto para este documento.",
|
||||
"resource-limit": "Este PDF é grande ou complexo demais para ser lido aqui. O arquivo em si está bem e continua abrindo."
|
||||
}
|
||||
},
|
||||
"loadError": "Não foi possível carregar este documento.",
|
||||
|
||||
@@ -9077,10 +9077,10 @@ model InboundDocument {
|
||||
/// Why the most recent index attempt produced NO index — a closed string
|
||||
/// enum (`DocumentIndexOutcomeValue` in
|
||||
/// `src/lib/validations/inbound-documents.ts`): local-empty |
|
||||
/// local-unsupported | decrypt-error | raster-failed | provider-error |
|
||||
/// pdf-needs-anthropic | empty-transcription. Null after a successful
|
||||
/// attempt (the `DocumentContentIndex` row itself is the success signal)
|
||||
/// and before the first attempt. Lets the detail view tell "never tried"
|
||||
/// local-unsupported | decrypt-error | resource-limit | raster-failed |
|
||||
/// provider-error | pdf-needs-anthropic | empty-transcription. Null after a
|
||||
/// successful attempt (the `DocumentContentIndex` row itself is the success
|
||||
/// signal) and before the first attempt. Lets the detail view tell "never tried"
|
||||
/// from "tried and here is why it failed" instead of showing a bare
|
||||
/// not-indexed state for both.
|
||||
lastIndexOutcome String? @map("last_index_outcome")
|
||||
|
||||
@@ -290,6 +290,36 @@ describe("indexDocumentContent — local fallback path", () => {
|
||||
expect(outcome).toEqual({ indexed: false, reason: "local-empty" });
|
||||
expect(upsertContentIndex).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("reports a PDF past the reader's limits as resource-limit, not as a file that would not open", async () => {
|
||||
vi.mocked(resolveDocumentVisionProvider).mockResolvedValue({
|
||||
chain: [],
|
||||
pick: null,
|
||||
} as never);
|
||||
vi.mocked(localExtractText).mockResolvedValue({
|
||||
ok: false,
|
||||
reason: "resource-limit",
|
||||
});
|
||||
|
||||
const outcome = await indexDocumentContent("user-1", "doc-1");
|
||||
expect(outcome).toEqual({ indexed: false, reason: "resource-limit" });
|
||||
expect(upsertContentIndex).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("keeps resource-limit over a provider note: the provider never got further either", async () => {
|
||||
vi.mocked(resolveDocumentVisionProvider).mockResolvedValue(PICK as never);
|
||||
vi.mocked(prepareVisionInput).mockReturnValue({
|
||||
ok: false,
|
||||
reason: "rasterFailed",
|
||||
} as never);
|
||||
vi.mocked(localExtractText).mockResolvedValue({
|
||||
ok: false,
|
||||
reason: "resource-limit",
|
||||
});
|
||||
|
||||
const outcome = await indexDocumentContent("user-1", "doc-1");
|
||||
expect(outcome).toEqual({ indexed: false, reason: "resource-limit" });
|
||||
});
|
||||
});
|
||||
|
||||
describe("indexDocumentContent — the documentAi capability", () => {
|
||||
|
||||
@@ -0,0 +1,53 @@
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
|
||||
/**
|
||||
* A PDF that crosses the reader's memory or time bound is reported as such
|
||||
* (`resource-limit`), not as a read error, which the index job records as a
|
||||
* stored file that would not open. The bound itself is the isolated reader's
|
||||
* (`../pdf-isolation.ts`); here it is the error it throws.
|
||||
*/
|
||||
|
||||
const { annotate, logCaught, readPdfText } = vi.hoisted(() => ({
|
||||
annotate: vi.fn(),
|
||||
logCaught: vi.fn(),
|
||||
readPdfText: vi.fn(),
|
||||
}));
|
||||
vi.mock("@/lib/logging/context", () => ({ annotate }));
|
||||
vi.mock("@/lib/logging/signal", () => ({ logCaught }));
|
||||
vi.mock("../pdfjs", async (importOriginal) => ({
|
||||
...(await importOriginal<typeof import("../pdfjs")>()),
|
||||
readPdfText,
|
||||
}));
|
||||
|
||||
import { PdfResourceLimitError } from "../pdfjs";
|
||||
import { extractPdfText, localExtractText } from "../local-extract";
|
||||
|
||||
describe("a PDF past the reader's limits", () => {
|
||||
it.each(["memory", "time", "exit"] as const)(
|
||||
"answers resource-limit for the %s bound, without the error log a broken file gets",
|
||||
async (limit) => {
|
||||
readPdfText.mockRejectedValueOnce(new PdfResourceLimitError(limit));
|
||||
annotate.mockClear();
|
||||
logCaught.mockClear();
|
||||
const bytes = Buffer.from("%PDF-1.7 large");
|
||||
expect(await extractPdfText(bytes)).toEqual({
|
||||
ok: false,
|
||||
reason: "resource-limit",
|
||||
});
|
||||
expect(logCaught).not.toHaveBeenCalled();
|
||||
expect(annotate).toHaveBeenCalledWith({
|
||||
action: { name: "documents.localIndex.resourceLimit" },
|
||||
meta: { limit, bytes: bytes.length },
|
||||
});
|
||||
},
|
||||
);
|
||||
|
||||
it("still answers error for anything else the reader throws", async () => {
|
||||
const broken = new Error("Invalid PDF structure");
|
||||
broken.name = "InvalidPDFException";
|
||||
readPdfText.mockRejectedValueOnce(broken);
|
||||
expect(
|
||||
await localExtractText(Buffer.from("%PDF-1.7"), "application/pdf"),
|
||||
).toEqual({ ok: false, reason: "error" });
|
||||
});
|
||||
});
|
||||
@@ -78,6 +78,7 @@ export type IndexFailureReason =
|
||||
| "local-empty"
|
||||
| "local-unsupported"
|
||||
| "decrypt-error"
|
||||
| "resource-limit"
|
||||
| "raster-failed"
|
||||
| "provider-error"
|
||||
| "pdf-needs-anthropic"
|
||||
@@ -342,6 +343,12 @@ async function tryLocalIndex(
|
||||
if (result.reason === "unsupported") {
|
||||
return { indexed: false, reason: "local-unsupported" };
|
||||
}
|
||||
if (result.reason === "resource-limit") {
|
||||
// Too large or too complex for the reader's bounds: said as such, and
|
||||
// final like the thumbnail's refusal of the same bytes. Nothing retries
|
||||
// it on its own; a person can still ask for a read.
|
||||
return { indexed: false, reason: "resource-limit" };
|
||||
}
|
||||
if (result.reason === "error") {
|
||||
return { indexed: false, reason: "decrypt-error" };
|
||||
}
|
||||
|
||||
@@ -32,7 +32,7 @@ import { Buffer } from "node:buffer";
|
||||
import { annotate } from "@/lib/logging/context";
|
||||
import { logCaught } from "@/lib/logging/signal";
|
||||
|
||||
import { readPdfText } from "./pdfjs";
|
||||
import { PdfResourceLimitError, readPdfText } from "./pdfjs";
|
||||
|
||||
/**
|
||||
* Minimum joined text length for a PDF to count as a real text-layer document.
|
||||
@@ -54,15 +54,23 @@ export type LocalExtractSource = "local-pdf" | "local-ocr";
|
||||
* a provider vision pass is the only way to read it.
|
||||
* - `unsupported` → a MIME the local path cannot read without OCR (images) or
|
||||
* at all; leave un-indexed locally.
|
||||
* - `resource-limit` → the PDF crossed the reader's memory or time bound
|
||||
* (`PdfResourceLimitError`): too large or too complex to
|
||||
* read here. The same bytes cross it again, so it is a
|
||||
* final answer, as it is for thumbnails.
|
||||
* - `error` → the file was malformed / unreadable; treat as un-indexed.
|
||||
*/
|
||||
export type LocalExtractResult =
|
||||
| { ok: true; text: string; source: LocalExtractSource }
|
||||
| { ok: false; reason: "empty" | "unsupported" | "error" };
|
||||
| {
|
||||
ok: false;
|
||||
reason: "empty" | "unsupported" | "resource-limit" | "error";
|
||||
};
|
||||
|
||||
/**
|
||||
* Extract a text-layer PDF's embedded text server-side. Returns `empty` for a
|
||||
* scanned/no-text-layer PDF and `error` for a malformed/encrypted one — never
|
||||
* scanned/no-text-layer PDF, `resource-limit` for one past the reader's memory
|
||||
* or time bound, and `error` for a malformed/encrypted one — never
|
||||
* throws, so a bad document can never abort the caller (the upload/backfill job
|
||||
* just leaves it un-indexed). An `error` leaves a `warn` line naming what
|
||||
* pdf.js threw, because the outcome alone does not say why a document stayed
|
||||
@@ -76,6 +84,15 @@ export async function extractPdfText(
|
||||
// Bound the page walk so a pathological many-page PDF costs no more.
|
||||
text = (await readPdfText(buffer, LOCAL_MAX_PAGES)).trim();
|
||||
} catch (err) {
|
||||
if (err instanceof PdfResourceLimitError) {
|
||||
// Not a fault of the file's bytes being unreadable: they are readable,
|
||||
// only not within the bounds this server gives one document.
|
||||
annotate({
|
||||
action: { name: "documents.localIndex.resourceLimit" },
|
||||
meta: { limit: err.limit, bytes: buffer.length },
|
||||
});
|
||||
return { ok: false, reason: "resource-limit" };
|
||||
}
|
||||
const errorType = err instanceof Error ? err.name || "Error" : typeof err;
|
||||
annotate({
|
||||
action: { name: "documents.localIndex.extractFailed" },
|
||||
|
||||
@@ -300,6 +300,7 @@ const inboundDocument = z
|
||||
"local-empty",
|
||||
"local-unsupported",
|
||||
"decrypt-error",
|
||||
"resource-limit",
|
||||
"raster-failed",
|
||||
"provider-error",
|
||||
"pdf-needs-anthropic",
|
||||
|
||||
@@ -115,6 +115,7 @@ export const DOCUMENT_INDEX_OUTCOMES = [
|
||||
"local-empty",
|
||||
"local-unsupported",
|
||||
"decrypt-error",
|
||||
"resource-limit",
|
||||
"raster-failed",
|
||||
"provider-error",
|
||||
"pdf-needs-anthropic",
|
||||
|
||||
Reference in new issue
Block a user