From 893db887e968ea92dbaf98fc2f8a5a02317a8016 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Thu, 13 Aug 2026 12:40:50 +0200 Subject: [PATCH] Record why a document's index attempt failed and say so A content-index attempt that failed (raster failure, a scan with no text layer, a provider error) recorded its reason only in the server's observability stream. In the vault, hasContentIndex false looked identical for a document nobody ever tried and one that failed three times, and the detail sheet had nothing to say about either. Apply the summaryState pattern to the index. Two additive columns on InboundDocument (migration 0333) record when the last attempt finished and why it produced no index, as a closed reason set: local-empty, local-unsupported, decrypt-error, raster-failed, provider-error, pdf-needs-anthropic, empty-transcription. The writer sits at the choke point the auto job and the corpus backfill share, and in the manual index route; a success clears the reason. The provider path's specific failure now survives the local fallback, since "no text" is less actionable than "the PDF would not render". rasterizePdf names its failure class so a build that cannot rasterize at all is distinguished from a PDF that would not render. New guard in both index paths: an empty provider transcription never becomes a successful empty index. The detail sheet renders the reason in plain words (muted, six locales). The DTO and OpenAPI carry the two fields; the disaster recovery backup and restore carry them verbatim like their summary siblings. Refs #776 --- docs/api/openapi.yaml | 40 ++- messages/de.json | 11 + messages/en.json | 11 + messages/es.json | 11 + messages/fr.json | 11 + messages/it.json | 11 + messages/pl.json | 11 + .../migration.sql | 10 + prisma/schema.prisma | 16 + .../api/admin/backups/[id]/restore/route.ts | 4 + .../api/documents/inbound/[id]/index/route.ts | 53 ++++ .../labs/ocr/extract/__tests__/route.test.ts | 5 +- .../__tests__/document-ai-section.test.tsx | 41 +++ .../__tests__/document-card.test.tsx | 2 + .../__tests__/document-facts-review.test.tsx | 2 + .../__tests__/episode-documents-card.test.tsx | 2 + .../documents/__tests__/vault-utils.test.ts | 2 + .../documents/document-ai-section.tsx | 24 ++ .../documents/document-detail-sheet.tsx | 1 + .../__tests__/index-outcome-record.test.ts | 278 ++++++++++++++++++ .../documents/__tests__/rasterize-pdf.test.ts | 7 +- src/lib/documents/ai-route-support.ts | 19 +- src/lib/documents/index-document.ts | 152 ++++++++-- src/lib/documents/rasterize-pdf.ts | 18 +- src/lib/documents/store.ts | 10 + src/lib/export/records-backup.ts | 9 + src/lib/openapi/routes/documents.ts | 14 +- src/lib/validations/backup.ts | 4 + src/lib/validations/inbound-documents.ts | 40 +++ 29 files changed, 786 insertions(+), 33 deletions(-) create mode 100644 src/lib/documents/__tests__/index-outcome-record.test.ts diff --git a/docs/api/openapi.yaml b/docs/api/openapi.yaml index 8a1a951f6..2ba7e1048 100644 --- a/docs/api/openapi.yaml +++ b/docs/api/openapi.yaml @@ -37857,6 +37857,22 @@ components: - local-pdf - local-ocr - type: "null" + lastIndexAttemptAt: + anyOf: + - type: string + - type: "null" + lastIndexOutcome: + anyOf: + - type: string + enum: + - local-empty + - local-unsupported + - decrypt-error + - raster-failed + - provider-error + - pdf-needs-anthropic + - empty-transcription + - type: "null" hasThumbnail: type: boolean createdAt: @@ -37882,6 +37898,8 @@ components: - servingClass - hasContentIndex - contentIndexSource + - lastIndexAttemptAt + - lastIndexOutcome - hasThumbnail - createdAt - updatedAt @@ -37893,7 +37911,9 @@ components: download-only (`attachment`); render/download by it, never by MIME guess. `hasContentIndex` is true when the document has a content-search index (auto-indexed on upload); `contentIndexSource` says how that index was produced — `vision` means an AI provider read the original, the other values are provider-free local - extractions, and it is null when `hasContentIndex` is false. `hasThumbnail` is true when a preview thumbnail has + extractions, and it is null when `hasContentIndex` is false. `lastIndexAttemptAt` is when the most recent index + attempt finished (successful or not; null when none ran); `lastIndexOutcome` says why that attempt produced no + index, and is null when the attempt succeeded or none ran. `hasThumbnail` is true when a preview thumbnail has been rendered in the background (fetch it from `/api/documents/inbound/{id}/thumbnail`); false means no preview yet or an unsupported type — show a placeholder. Treat an unknown `kind` value as OTHER when decoding. DocumentConditionLink: @@ -38205,6 +38225,22 @@ components: - local-pdf - local-ocr - type: "null" + lastIndexAttemptAt: + anyOf: + - type: string + - type: "null" + lastIndexOutcome: + anyOf: + - type: string + enum: + - local-empty + - local-unsupported + - decrypt-error + - raster-failed + - provider-error + - pdf-needs-anthropic + - empty-transcription + - type: "null" hasThumbnail: type: boolean createdAt: @@ -38250,6 +38286,8 @@ components: - servingClass - hasContentIndex - contentIndexSource + - lastIndexAttemptAt + - lastIndexOutcome - hasThumbnail - createdAt - updatedAt diff --git a/messages/de.json b/messages/de.json index 779d21e65..c9ee37e66 100644 --- a/messages/de.json +++ b/messages/de.json @@ -9506,6 +9506,17 @@ "generate": "Zusammenfassung erstellen", "generating": "Wird erstellt…" }, + "index": { + "outcome": { + "local-empty": "Es wurde kein lesbarer Text gefunden. Das sieht nach einem Scan aus; ein KI-Vision-Anbieter kann ihn lesen.", + "local-unsupported": "Dieser Dateityp kann für die Suche nicht gelesen werden.", + "decrypt-error": "Die gespeicherte Datei konnte zum Lesen nicht geöffnet werden.", + "raster-failed": "Die PDF-Seiten konnten zum Lesen nicht gerendert werden.", + "provider-error": "Der KI-Anbieter konnte dieses Dokument beim letzten Versuch nicht lesen.", + "pdf-needs-anthropic": "Der eingerichtete KI-Anbieter kann keine PDF-Dateien lesen. Ein Claude-Anbieter oder lokale OCR kann es.", + "empty-transcription": "Der KI-Anbieter hat für dieses Dokument keinen Text geliefert." + } + }, "loadError": "Dieses Dokument konnte nicht geladen werden.", "saveError": "Die Änderung konnte nicht gespeichert werden.", "deleteError": "Das Dokument konnte nicht gelöscht werden.", diff --git a/messages/en.json b/messages/en.json index f417c2673..865889b30 100644 --- a/messages/en.json +++ b/messages/en.json @@ -9506,6 +9506,17 @@ "generate": "Generate summary", "generating": "Generating…" }, + "index": { + "outcome": { + "local-empty": "No readable text was found. This looks like a scan; an AI vision provider can read it.", + "local-unsupported": "This file type can't be read for search.", + "decrypt-error": "The stored file couldn't be opened for reading.", + "raster-failed": "The PDF pages couldn't be rendered for reading.", + "provider-error": "The AI provider couldn't read this document on the last try.", + "pdf-needs-anthropic": "The configured AI provider can't read PDF files. A Claude provider or local OCR can.", + "empty-transcription": "The AI provider returned no text for this document." + } + }, "loadError": "Couldn't load this document.", "saveError": "Couldn't save the change.", "deleteError": "Couldn't delete the document.", diff --git a/messages/es.json b/messages/es.json index 09bea375a..91a02a17a 100644 --- a/messages/es.json +++ b/messages/es.json @@ -9506,6 +9506,17 @@ "generate": "Generar resumen", "generating": "Generando…" }, + "index": { + "outcome": { + "local-empty": "No se encontró texto legible. Parece un documento escaneado; un proveedor de visión de IA puede leerlo.", + "local-unsupported": "Este tipo de archivo no se puede leer para la búsqueda.", + "decrypt-error": "El archivo guardado no se pudo abrir para su lectura.", + "raster-failed": "Las páginas del PDF no se pudieron procesar para su lectura.", + "provider-error": "El proveedor de IA no pudo leer este documento en el último intento.", + "pdf-needs-anthropic": "El proveedor de IA configurado no puede leer archivos PDF. Un proveedor Claude o el OCR local sí pueden.", + "empty-transcription": "El proveedor de IA no devolvió texto para este documento." + } + }, "loadError": "No se pudo cargar este documento.", "saveError": "No se pudo guardar el cambio.", "deleteError": "No se pudo eliminar el documento.", diff --git a/messages/fr.json b/messages/fr.json index f55859fba..fa5c95084 100644 --- a/messages/fr.json +++ b/messages/fr.json @@ -9506,6 +9506,17 @@ "generate": "Générer le résumé", "generating": "Génération…" }, + "index": { + "outcome": { + "local-empty": "Aucun texte lisible n’a été trouvé. Cela ressemble à un scan ; un fournisseur de vision IA peut le lire.", + "local-unsupported": "Ce type de fichier ne peut pas être lu pour la recherche.", + "decrypt-error": "Le fichier stocké n’a pas pu être ouvert pour la lecture.", + "raster-failed": "Les pages du PDF n’ont pas pu être rendues pour la lecture.", + "provider-error": "Le fournisseur d’IA n’a pas pu lire ce document lors de la dernière tentative.", + "pdf-needs-anthropic": "Le fournisseur d’IA configuré ne peut pas lire les fichiers PDF. Un fournisseur Claude ou l’OCR local le peut.", + "empty-transcription": "Le fournisseur d’IA n’a renvoyé aucun texte pour ce document." + } + }, "loadError": "Impossible de charger ce document.", "saveError": "Impossible d'enregistrer la modification.", "deleteError": "Impossible de supprimer le document.", diff --git a/messages/it.json b/messages/it.json index 844cdb904..6786af624 100644 --- a/messages/it.json +++ b/messages/it.json @@ -9506,6 +9506,17 @@ "generate": "Genera riepilogo", "generating": "Generazione…" }, + "index": { + "outcome": { + "local-empty": "Non è stato trovato testo leggibile. Sembra una scansione; un provider di visione IA può leggerla.", + "local-unsupported": "Questo tipo di file non può essere letto per la ricerca.", + "decrypt-error": "Il file archiviato non può essere aperto per la lettura.", + "raster-failed": "Le pagine del PDF non possono essere elaborate per la lettura.", + "provider-error": "Il provider IA non è riuscito a leggere questo documento all’ultimo tentativo.", + "pdf-needs-anthropic": "Il provider IA configurato non può leggere i file PDF. Un provider Claude o l’OCR locale sì.", + "empty-transcription": "Il provider IA non ha restituito testo per questo documento." + } + }, "loadError": "Impossibile caricare questo documento.", "saveError": "Impossibile salvare la modifica.", "deleteError": "Impossibile eliminare il documento.", diff --git a/messages/pl.json b/messages/pl.json index 082ee3bac..07c5e2a05 100644 --- a/messages/pl.json +++ b/messages/pl.json @@ -9506,6 +9506,17 @@ "generate": "Wygeneruj podsumowanie", "generating": "Generowanie…" }, + "index": { + "outcome": { + "local-empty": "Nie znaleziono czytelnego tekstu. To wygląda na skan; dostawca wizji AI może go odczytać.", + "local-unsupported": "Tego typu pliku nie można odczytać na potrzeby wyszukiwania.", + "decrypt-error": "Nie udało się otworzyć zapisanego pliku do odczytu.", + "raster-failed": "Nie udało się przetworzyć stron PDF do odczytu.", + "provider-error": "Dostawca AI nie zdołał odczytać tego dokumentu przy ostatniej próbie.", + "pdf-needs-anthropic": "Skonfigurowany dostawca AI nie odczytuje plików PDF. Poradzi sobie z tym dostawca Claude lub lokalny OCR.", + "empty-transcription": "Dostawca AI nie zwrócił tekstu dla tego dokumentu." + } + }, "loadError": "Nie udało się wczytać tego dokumentu.", "saveError": "Nie udało się zapisać zmiany.", "deleteError": "Nie udało się usunąć dokumentu.", diff --git a/prisma/migrations/0333_summary_pending_heal_index_outcome/migration.sql b/prisma/migrations/0333_summary_pending_heal_index_outcome/migration.sql index 9acfc02ee..5740eca26 100644 --- a/prisma/migrations/0333_summary_pending_heal_index_outcome/migration.sql +++ b/prisma/migrations/0333_summary_pending_heal_index_outcome/migration.sql @@ -10,3 +10,13 @@ UPDATE "inbound_documents" SET "summary_state" = 'NONE' WHERE "summary_state" = 'PENDING'; + +-- Refs #776 — record what became of a document's content-index attempt. +-- Additive, nullable, no backfill: existing rows read NULL ("no attempt +-- recorded"), which is exactly true for them — outcomes only exist from the +-- release that writes them. `last_index_outcome` is a closed string enum +-- enforced in code (see `DocumentIndexOutcomeValue`); NULL after a +-- successful attempt, since the `document_content_index` row itself is the +-- success signal. +ALTER TABLE "inbound_documents" ADD COLUMN "last_index_attempt_at" TIMESTAMP(3); +ALTER TABLE "inbound_documents" ADD COLUMN "last_index_outcome" TEXT; diff --git a/prisma/schema.prisma b/prisma/schema.prisma index 6b3692997..25c7d873d 100644 --- a/prisma/schema.prisma +++ b/prisma/schema.prisma @@ -7536,6 +7536,22 @@ model InboundDocument { /// and one mid-flight all leave it null. NONE is the default, which keeps /// every pre-existing row correct — none of them were ever attempted. summaryState DocumentSummaryState @default(NONE) @map("summary_state") + /// When the most recent content-index attempt finished (auto job, corpus + /// backfill, or the manual index route — they share one recorder), + /// successful or not. Null until any attempt ran. Diagnostic state the + /// detail view pairs with `lastIndexOutcome`; carried verbatim in the + /// disaster-recovery backup like its summary siblings. + lastIndexAttemptAt DateTime? @map("last_index_attempt_at") + /// Why the most recent index attempt produced NO index — a closed string + /// enum (`DocumentIndexOutcomeValue` in + /// `src/lib/validations/inbound-documents.ts`): local-empty | + /// local-unsupported | decrypt-error | raster-failed | provider-error | + /// pdf-needs-anthropic | empty-transcription. Null after a successful + /// attempt (the `DocumentContentIndex` row itself is the success signal) + /// and before the first attempt. Lets the detail view tell "never tried" + /// from "tried and here is why it failed" instead of showing a bare + /// not-indexed state for both. + lastIndexOutcome String? @map("last_index_outcome") facts ExtractedFact[] conditionLinks DocumentConditionLink[] diff --git a/src/app/api/admin/backups/[id]/restore/route.ts b/src/app/api/admin/backups/[id]/restore/route.ts index 52dc674fa..6bfb9b39b 100644 --- a/src/app/api/admin/backups/[id]/restore/route.ts +++ b/src/app/api/admin/backups/[id]/restore/route.ts @@ -1319,6 +1319,10 @@ const handler = apiHandler( ? new Date(document.summaryGeneratedAt) : null, summaryState: document.summaryState as never, + lastIndexAttemptAt: document.lastIndexAttemptAt + ? new Date(document.lastIndexAttemptAt) + : null, + lastIndexOutcome: document.lastIndexOutcome ?? null, createdAt: new Date(document.createdAt!), updatedAt: new Date(document.updatedAt!), })), diff --git a/src/app/api/documents/inbound/[id]/index/route.ts b/src/app/api/documents/inbound/[id]/index/route.ts index 072271263..d64166fe1 100644 --- a/src/app/api/documents/inbound/[id]/index/route.ts +++ b/src/app/api/documents/inbound/[id]/index/route.ts @@ -49,6 +49,7 @@ import { } from "@/lib/documents/ai-route-support"; import { maybeAutoStageLabFacts } from "@/lib/documents/auto-stage-labs"; import { upsertContentIndex } from "@/lib/documents/content-index"; +import { recordIndexAttempt } from "@/lib/documents/index-document"; import { DocumentDescribeError, transcribeDocument, @@ -93,6 +94,15 @@ async function finishIndex( source: "vision" | "text-ocr", tokenCount: number, ): Promise { + // Refs #776 — the manual route is the second writer of the attempt record + // (the auto job + backfill share `indexLoadedDocument`): a success clears + // any stored failure reason so the detail view stops explaining a problem + // that no longer exists. + await recordIndexAttempt(userId, documentId, { + indexed: true, + source, + tokenCount, + }); await auditLog("documents.inbound.index", { userId, ipAddress: getClientIp(request), @@ -183,21 +193,44 @@ async function handleVisionIndex( const vision = await prepareVisionInput(document, pick.pdfSupported); if (!vision.ok) { // Preparation failed before any provider dispatch — the slot goes back. + // Refs #776 — each failure is also recorded on the row so the detail view + // can explain the missing index after the toast is gone. await refundDocumentAiSlot(userId); if (vision.reason === "pdfNeedsAnthropic") { + await recordIndexAttempt(userId, document.id, { + indexed: false, + reason: "pdf-needs-anthropic", + }); return apiError( "PDF scanning needs a Claude vision provider; use local OCR instead.", 422, { errorCode: "documents.inbound.pdfNeedsAnthropic" }, ); } + if (vision.reason === "rasterFailed") { + await recordIndexAttempt(userId, document.id, { + indexed: false, + reason: "raster-failed", + }); + return apiError("The PDF pages couldn't be rendered for scanning.", 422, { + errorCode: "documents.inbound.extractFailed", + }); + } if (vision.reason === "fileType") { + await recordIndexAttempt(userId, document.id, { + indexed: false, + reason: "local-unsupported", + }); return apiError( "This document can't be scanned. Use local OCR (text mode).", 422, { errorCode: "documents.inbound.fileType" }, ); } + await recordIndexAttempt(userId, document.id, { + indexed: false, + reason: "decrypt-error", + }); return apiError("Couldn't read the stored document.", 422, { errorCode: "documents.inbound.extractFailed", }); @@ -230,6 +263,22 @@ async function handleVisionIndex( reservation.reserved, dateKey, ); + // Refs #776 — the empty-transcription guard, same contract as the auto + // path (`tryProviderIndex`): a provider answer with no text must never + // become a "successful" empty index. The spend stays charged (the + // provider was called); the honest answer is an error plus the recorded + // reason. + if (text.trim().length === 0) { + await recordIndexAttempt(userId, document.id, { + indexed: false, + reason: "empty-transcription", + }); + return apiError( + "The provider returned no text for this document. Try a clearer copy.", + 422, + { errorCode: "documents.inbound.extractFailed" }, + ); + } const { tokenCount } = await upsertContentIndex({ userId, documentId: document.id, @@ -240,6 +289,10 @@ async function handleVisionIndex( return finishIndex(request, userId, document.id, "vision", tokenCount); } catch (err) { await reconcileSpend(userId, reservation.reserved, 0, dateKey); + await recordIndexAttempt(userId, document.id, { + indexed: false, + reason: "provider-error", + }); if (err instanceof DocumentDescribeError) { return apiError("Couldn't read the document. Try a clearer copy.", 422, { errorCode: "documents.inbound.extractFailed", diff --git a/src/app/api/labs/ocr/extract/__tests__/route.test.ts b/src/app/api/labs/ocr/extract/__tests__/route.test.ts index 5f1f4583d..f5358fa51 100644 --- a/src/app/api/labs/ocr/extract/__tests__/route.test.ts +++ b/src/app/api/labs/ocr/extract/__tests__/route.test.ts @@ -196,7 +196,10 @@ describe("POST /api/labs/ocr/extract — vision PDF rasterization", () => { }); it("falls back to pdfNeedsAnthropic when rasterization fails", async () => { - vi.mocked(rasterizePdf).mockResolvedValue({ ok: false }); + vi.mocked(rasterizePdf).mockResolvedValue({ + ok: false, + reason: "render-failed", + }); const res = await POST(pdfReq()); expect(res.status).toBe(422); diff --git a/src/components/documents/__tests__/document-ai-section.test.tsx b/src/components/documents/__tests__/document-ai-section.test.tsx index 9eecdbefb..c0af03212 100644 --- a/src/components/documents/__tests__/document-ai-section.test.tsx +++ b/src/components/documents/__tests__/document-ai-section.test.tsx @@ -213,3 +213,44 @@ describe("", () => { expect(html).toContain("Summarise"); }); }); + +describe(" index outcome (#776)", () => { + it("explains a failed index attempt in plain words, muted", () => { + const html = render( + base({ hasContentIndex: false, lastIndexOutcome: "local-empty" }), + ); + expect(html).toContain('data-slot="document-index-outcome"'); + expect(html).toContain("No readable text was found"); + expect(html).toContain("text-muted-foreground"); + }); + + it("also explains it on the auto-read unread branch", () => { + const html = render( + base({ + autoReadEnabled: true, + hasContentIndex: false, + lastIndexOutcome: "raster-failed", + }), + ); + expect(html).toContain('data-slot="document-index-outcome"'); + expect(html).toContain("couldn't be rendered"); + }); + + it("says nothing once the document IS indexed (a past failure is history)", () => { + const html = render( + base({ + hasContentIndex: true, + contentIndexSource: "vision", + lastIndexOutcome: "provider-error", + }), + ); + expect(html).not.toContain('data-slot="document-index-outcome"'); + }); + + it("says nothing when no attempt ever ran (never tried is not a failure)", () => { + const html = render( + base({ hasContentIndex: false, lastIndexOutcome: null }), + ); + expect(html).not.toContain('data-slot="document-index-outcome"'); + }); +}); diff --git a/src/components/documents/__tests__/document-card.test.tsx b/src/components/documents/__tests__/document-card.test.tsx index 0d4aab26a..7b0aa82a1 100644 --- a/src/components/documents/__tests__/document-card.test.tsx +++ b/src/components/documents/__tests__/document-card.test.tsx @@ -34,6 +34,8 @@ function doc(overrides: Partial = {}): InboundDocumentDto { servingClass: "inline", hasContentIndex: false, contentIndexSource: null, + lastIndexAttemptAt: null, + lastIndexOutcome: null, hasThumbnail: false, createdAt: "2025-10-05T08:00:00.000Z", updatedAt: "2025-10-05T08:00:00.000Z", diff --git a/src/components/documents/__tests__/document-facts-review.test.tsx b/src/components/documents/__tests__/document-facts-review.test.tsx index a90b630b5..75dfb3f7b 100644 --- a/src/components/documents/__tests__/document-facts-review.test.tsx +++ b/src/components/documents/__tests__/document-facts-review.test.tsx @@ -85,6 +85,8 @@ function doc( servingClass: "inline", hasContentIndex: true, contentIndexSource: "vision", + lastIndexAttemptAt: null, + lastIndexOutcome: null, hasThumbnail: false, createdAt: "2026-08-01T10:00:00.000Z", updatedAt: "2026-08-01T10:00:00.000Z", diff --git a/src/components/documents/__tests__/episode-documents-card.test.tsx b/src/components/documents/__tests__/episode-documents-card.test.tsx index 2d5547c69..db5856609 100644 --- a/src/components/documents/__tests__/episode-documents-card.test.tsx +++ b/src/components/documents/__tests__/episode-documents-card.test.tsx @@ -54,6 +54,8 @@ function doc(id: string, title: string): InboundDocumentDto { servingClass: "inline", hasContentIndex: false, contentIndexSource: null, + lastIndexAttemptAt: null, + lastIndexOutcome: null, hasThumbnail: false, createdAt: "2025-10-05T08:00:00.000Z", updatedAt: "2025-10-05T08:00:00.000Z", diff --git a/src/components/documents/__tests__/vault-utils.test.ts b/src/components/documents/__tests__/vault-utils.test.ts index 6947140aa..201963f85 100644 --- a/src/components/documents/__tests__/vault-utils.test.ts +++ b/src/components/documents/__tests__/vault-utils.test.ts @@ -42,6 +42,8 @@ function doc(overrides: Partial = {}): InboundDocumentDto { servingClass: "inline", hasContentIndex: false, contentIndexSource: null, + lastIndexAttemptAt: null, + lastIndexOutcome: null, hasThumbnail: false, createdAt: "2026-03-11T08:00:00.000Z", updatedAt: "2026-03-11T08:00:00.000Z", diff --git a/src/components/documents/document-ai-section.tsx b/src/components/documents/document-ai-section.tsx index 0e85d1e05..33f19e58b 100644 --- a/src/components/documents/document-ai-section.tsx +++ b/src/components/documents/document-ai-section.tsx @@ -38,6 +38,7 @@ import { cn } from "@/lib/utils"; import { isAiReadSource, type DocumentContentIndexSourceValue, + type DocumentIndexOutcomeValue, type DocumentSuggestionDto, type DocumentSummaryMode, } from "@/lib/validations/inbound-documents"; @@ -74,6 +75,7 @@ export function DocumentAiSection({ onCloseSummary, hasContentIndex, contentIndexSource, + lastIndexOutcome = null, indexPending, onIndex, }: { @@ -111,11 +113,24 @@ export function DocumentAiSection({ onCloseSummary: () => void; hasContentIndex: boolean; contentIndexSource: DocumentContentIndexSourceValue | null; + /** + * Refs #776 — why the last index attempt produced no index (null: never + * tried, or the document IS indexed). Rendered as a muted plain-words line + * beside the read action so a failed read is explainable, not a mystery. + */ + lastIndexOutcome?: DocumentIndexOutcomeValue | null; indexPending: boolean; onIndex: () => void; }) { const { t } = useTranslations(); const aiRead = hasContentIndex && isAiReadSource(contentIndexSource); + // Refs #776 — only an un-indexed document explains itself; once an index + // exists the last failure is history. Muted per the text-colour rule: this + // is meta about the document, not content. + const indexOutcomeText = + !hasContentIndex && lastIndexOutcome + ? t(`documents.detail.index.outcome.${lastIndexOutcome}`) + : null; const readLabel = indexPending ? t("documents.ai.reading") : aiRead @@ -233,6 +248,15 @@ export function DocumentAiSection({ ) : null} + {indexOutcomeText ? ( +

+ {indexOutcomeText} +

+ ) : null} + {suggestErrorKey ? (

{suggestErrorKey} diff --git a/src/components/documents/document-detail-sheet.tsx b/src/components/documents/document-detail-sheet.tsx index fd37ddc07..1559d1c53 100644 --- a/src/components/documents/document-detail-sheet.tsx +++ b/src/components/documents/document-detail-sheet.tsx @@ -875,6 +875,7 @@ export function DocumentDetailSheet({ }} hasContentIndex={doc.hasContentIndex} contentIndexSource={doc.contentIndexSource} + lastIndexOutcome={doc.lastIndexOutcome} indexPending={indexDoc.isPending} onIndex={runIndex} /> diff --git a/src/lib/documents/__tests__/index-outcome-record.test.ts b/src/lib/documents/__tests__/index-outcome-record.test.ts new file mode 100644 index 000000000..c11d98e2b --- /dev/null +++ b/src/lib/documents/__tests__/index-outcome-record.test.ts @@ -0,0 +1,278 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; + +/** + * Refs #776 — an index attempt that ends without an index must record why. + * + * Before this, a failed attempt (raster fail, empty scan, provider error) + * left its reason only in wide-events: `hasContentIndex=false` was + * indistinguishable from "never tried". Pins: + * + * - `indexLoadedDocument` (the choke point the job AND the backfill share) + * records every attempt on the row: `lastIndexAttemptAt` always, + * `lastIndexOutcome` = the failure reason, or null on success. + * - The provider path's specific failure (raster-failed / + * pdf-needs-anthropic / provider-error / empty-transcription) survives + * the local fallback: when local can only say "no text", the recorded + * reason is the provider path's more actionable one. + * - An empty provider transcription NEVER becomes a "successful" empty + * index — it falls through to local instead. + */ + +vi.mock("@/lib/db", () => ({ + prisma: { + inboundDocument: { updateMany: vi.fn() }, + }, +})); +vi.mock("@/lib/documents/ai-route-support", () => ({ + loadOwnedDocument: vi.fn(), + prepareVisionInput: vi.fn(), +})); +vi.mock("@/lib/documents/content-index", () => ({ + upsertContentIndex: vi.fn().mockResolvedValue({ tokenCount: 5 }), +})); +vi.mock("@/lib/documents/describe", () => ({ + transcribeDocument: vi.fn(), +})); +vi.mock("@/lib/documents/local-extract", () => ({ + localExtractText: vi.fn(), +})); +vi.mock("@/lib/documents/store", () => ({ + decryptDocumentContent: vi.fn(() => Buffer.from([1, 2, 3])), +})); +vi.mock("@/lib/labs/ocr-upload", () => ({ + detectOcrMimeType: vi.fn(() => "application/pdf"), +})); +vi.mock("@/lib/documents/provider-order", () => ({ + resolveDocumentVisionProvider: vi.fn(), +})); +vi.mock("@/lib/documents/document-settings", () => ({ + documentAutoReadEnabled: vi.fn(), +})); +vi.mock("@/lib/ai/consent-guard", () => ({ + isExternalDocumentEgress: (providerType: string) => providerType !== "local", +})); +vi.mock("@/lib/ai/coach/budget", () => ({ + buildDateKey: vi.fn(() => "2026-08-13"), + reserveBudget: vi.fn(), + reconcileSpend: vi.fn().mockResolvedValue(undefined), + resolveDailyCap: vi.fn(() => 1000), +})); +vi.mock("@/lib/ai/ai-budgets", () => ({ + AI_BUDGETS: { documentTranscribe: { temperature: 0, maxTokens: 4000 } }, +})); + +import { indexDocumentContent, recordIndexAttempt } from "../index-document"; +import { prisma } from "@/lib/db"; +import { + loadOwnedDocument, + prepareVisionInput, +} from "@/lib/documents/ai-route-support"; +import { upsertContentIndex } from "@/lib/documents/content-index"; +import { transcribeDocument } from "@/lib/documents/describe"; +import { documentAutoReadEnabled } from "@/lib/documents/document-settings"; +import { localExtractText } from "@/lib/documents/local-extract"; +import { resolveDocumentVisionProvider } from "@/lib/documents/provider-order"; +import { reserveBudget } from "@/lib/ai/coach/budget"; + +const DOC = { + id: "doc-1", + kind: "OTHER", + contentEncrypted: new Uint8Array([1, 2, 3]), + contentCodec: "binary2", + mimeType: "application/pdf", + status: "STORED", +}; + +const PICK = { + chain: [{ providerType: "anthropic", instance: {} }], + pick: { + entry: { providerType: "anthropic", instance: {} }, + providerType: "anthropic", + pdfSupported: true, + }, +}; + +function visionOk() { + vi.mocked(prepareVisionInput).mockResolvedValue({ + ok: true, + images: [], + documents: [{ mediaType: "application/pdf", dataBase64: "AA==" }], + } as never); +} + +function localEmpty() { + vi.mocked(localExtractText).mockResolvedValue({ + ok: false, + reason: "empty", + } as never); +} + +function lastRecordedOutcome(): unknown { + const calls = vi.mocked(prisma.inboundDocument.updateMany).mock.calls; + const withOutcome = calls.filter( + ([arg]) => arg.data && "lastIndexOutcome" in (arg.data as object), + ); + const last = withOutcome[withOutcome.length - 1]; + return last ? (last[0].data as { lastIndexOutcome: unknown }) : undefined; +} + +beforeEach(() => { + vi.clearAllMocks(); + vi.mocked(loadOwnedDocument).mockResolvedValue(DOC as never); + vi.mocked(documentAutoReadEnabled).mockResolvedValue(true); + vi.mocked(resolveDocumentVisionProvider).mockResolvedValue(PICK as never); + vi.mocked(reserveBudget).mockResolvedValue({ + allowed: true, + reserved: 1, + } as never); + vi.mocked(localExtractText).mockResolvedValue({ + ok: true, + text: "glucose fasting creatinine values report", + source: "local-pdf", + } as never); + vi.mocked(prisma.inboundDocument.updateMany).mockResolvedValue({ + count: 1, + } as never); +}); + +describe("recordIndexAttempt — the row-level writer", () => { + it("records a failure reason with the attempt instant, owner-scoped", async () => { + await recordIndexAttempt("user-1", "doc-1", { + indexed: false, + reason: "local-empty", + }); + expect(prisma.inboundDocument.updateMany).toHaveBeenCalledWith({ + where: { id: "doc-1", userId: "user-1", deletedAt: null }, + data: { + lastIndexAttemptAt: expect.any(Date), + lastIndexOutcome: "local-empty", + }, + }); + }); + + it("clears the outcome on a successful attempt (the index row is the success signal)", async () => { + await recordIndexAttempt("user-1", "doc-1", { + indexed: true, + source: "vision", + tokenCount: 5, + }); + expect(prisma.inboundDocument.updateMany).toHaveBeenCalledWith( + expect.objectContaining({ + data: expect.objectContaining({ lastIndexOutcome: null }), + }), + ); + }); + + it("writes nothing for not-found (no row to explain) and never throws", async () => { + await recordIndexAttempt("user-1", "missing", { + indexed: false, + reason: "not-found", + }); + expect(prisma.inboundDocument.updateMany).not.toHaveBeenCalled(); + + vi.mocked(prisma.inboundDocument.updateMany).mockRejectedValueOnce( + new Error("db down"), + ); + await expect( + recordIndexAttempt("user-1", "doc-1", { + indexed: false, + reason: "local-empty", + }), + ).resolves.toBeUndefined(); + }); +}); + +describe("indexDocumentContent — the attempt record at the choke point", () => { + it("records the outcome on failure (job + backfill share this path)", async () => { + vi.mocked(resolveDocumentVisionProvider).mockResolvedValue({ + chain: [], + pick: null, + } as never); + localEmpty(); + + await indexDocumentContent("user-1", "doc-1"); + expect(lastRecordedOutcome()).toMatchObject({ + lastIndexOutcome: "local-empty", + }); + }); + + it("records a cleared outcome on success", async () => { + visionOk(); + vi.mocked(transcribeDocument).mockResolvedValue({ + text: "haemoglobin 14.2 cholesterol 190", + } as never); + + await indexDocumentContent("user-1", "doc-1"); + expect(lastRecordedOutcome()).toMatchObject({ lastIndexOutcome: null }); + }); +}); + +describe("indexDocumentContent — the provider path's reason survives the local fallback", () => { + it("provider-error: the provider threw and local had no text", async () => { + visionOk(); + vi.mocked(transcribeDocument).mockRejectedValueOnce(new Error("boom")); + localEmpty(); + + const outcome = await indexDocumentContent("user-1", "doc-1"); + expect(outcome).toEqual({ indexed: false, reason: "provider-error" }); + }); + + it("raster-failed: the PDF could not be rendered and local had no text", async () => { + vi.mocked(prepareVisionInput).mockResolvedValue({ + ok: false, + reason: "rasterFailed", + } as never); + localEmpty(); + + const outcome = await indexDocumentContent("user-1", "doc-1"); + expect(outcome).toEqual({ indexed: false, reason: "raster-failed" }); + }); + + it("pdf-needs-anthropic: the provider cannot read PDFs and local had no text", async () => { + vi.mocked(prepareVisionInput).mockResolvedValue({ + ok: false, + reason: "pdfNeedsAnthropic", + } as never); + localEmpty(); + + const outcome = await indexDocumentContent("user-1", "doc-1"); + expect(outcome).toEqual({ indexed: false, reason: "pdf-needs-anthropic" }); + }); + + it("a plain local failure keeps its own reason when the provider path had none", async () => { + vi.mocked(resolveDocumentVisionProvider).mockResolvedValue({ + chain: [], + pick: null, + } as never); + localEmpty(); + + const outcome = await indexDocumentContent("user-1", "doc-1"); + expect(outcome).toEqual({ indexed: false, reason: "local-empty" }); + }); +}); + +describe("indexDocumentContent — the empty-transcription guard", () => { + it("an empty provider answer NEVER becomes a successful empty index", async () => { + visionOk(); + vi.mocked(transcribeDocument).mockResolvedValue({ text: " " } as never); + localEmpty(); + + const outcome = await indexDocumentContent("user-1", "doc-1"); + expect(outcome).toEqual({ + indexed: false, + reason: "empty-transcription", + }); + expect(upsertContentIndex).not.toHaveBeenCalled(); + }); + + it("an empty provider answer still lands the LOCAL text when there is one", async () => { + visionOk(); + vi.mocked(transcribeDocument).mockResolvedValue({ text: "" } as never); + + const outcome = await indexDocumentContent("user-1", "doc-1"); + expect(outcome).toMatchObject({ indexed: true, source: "local-pdf" }); + expect(upsertContentIndex).toHaveBeenCalledWith( + expect.objectContaining({ source: "local-pdf" }), + ); + }); +}); diff --git a/src/lib/documents/__tests__/rasterize-pdf.test.ts b/src/lib/documents/__tests__/rasterize-pdf.test.ts index 6d8fb89e7..a00340e8a 100644 --- a/src/lib/documents/__tests__/rasterize-pdf.test.ts +++ b/src/lib/documents/__tests__/rasterize-pdf.test.ts @@ -102,10 +102,13 @@ describe("rasterizePdf", () => { it("returns { ok: false } and never throws on malformed input", async () => { await expect( rasterizePdf(Buffer.from("this is not a pdf")), - ).resolves.toEqual({ ok: false }); + ).resolves.toEqual({ ok: false, reason: "render-failed" }); }); it("returns { ok: false } on an empty buffer", async () => { - await expect(rasterizePdf(Buffer.alloc(0))).resolves.toEqual({ ok: false }); + await expect(rasterizePdf(Buffer.alloc(0))).resolves.toEqual({ + ok: false, + reason: "render-failed", + }); }); }); diff --git a/src/lib/documents/ai-route-support.ts b/src/lib/documents/ai-route-support.ts index e294000fe..49cf6a6fb 100644 --- a/src/lib/documents/ai-route-support.ts +++ b/src/lib/documents/ai-route-support.ts @@ -143,7 +143,15 @@ export type VisionInput = }[]; documents: { mediaType: "application/pdf"; dataBase64: string }[]; } - | { ok: false; reason: "decryptFailed" | "fileType" | "pdfNeedsAnthropic" }; + | { + ok: false; + // Refs #776 — `rasterFailed` (this PDF would not render on a raster- + // capable build) is distinct from `pdfNeedsAnthropic` (this build / + // provider cannot take a PDF at all): they need different fixes, so + // they must not share a label. + reason: + "decryptFailed" | "fileType" | "pdfNeedsAnthropic" | "rasterFailed"; + }; /** * Decrypt the stored original, re-derive its MIME from the bytes (never trust @@ -197,7 +205,14 @@ export async function prepareVisionInput( if (raster.ok) { return { ok: true, images: raster.images, documents: [] }; } - return { ok: false, reason: "pdfNeedsAnthropic" }; + // Refs #776 — a build that cannot rasterize at all is a capability gap a + // PDF-native (Anthropic) provider would cover; a render failure on a + // capable build is a property of THIS document. Report them apart. + return { + ok: false, + reason: + raster.reason === "unsupported" ? "pdfNeedsAnthropic" : "rasterFailed", + }; } return { diff --git a/src/lib/documents/index-document.ts b/src/lib/documents/index-document.ts index 319dae074..16f012f6a 100644 --- a/src/lib/documents/index-document.ts +++ b/src/lib/documents/index-document.ts @@ -52,22 +52,71 @@ import { documentAutoReadEnabled } from "@/lib/documents/document-settings"; import { localExtractText } from "@/lib/documents/local-extract"; import { resolveDocumentVisionProvider } from "@/lib/documents/provider-order"; import { decryptDocumentContent } from "@/lib/documents/store"; +import { prisma } from "@/lib/db"; +import { annotate } from "@/lib/logging/context"; import { detectOcrMimeType } from "@/lib/labs/ocr-upload"; import type { ProviderChainResolved } from "@/lib/ai/provider-runner"; +/** + * Why one document's index attempt ended without an index. Refs #776 — the + * provider path's specific failures are terminal reasons now: when the local + * fallback can only say "no text", the recorded reason is the provider path's + * more actionable one (a raster failure, a provider error, an empty + * transcription, or a PDF the picked provider cannot take), so the detail + * view can say WHY instead of a bare not-indexed. + */ +export type IndexFailureReason = + | "not-found" + | "local-empty" + | "local-unsupported" + | "decrypt-error" + | "raster-failed" + | "provider-error" + | "pdf-needs-anthropic" + | "empty-transcription"; + /** * Outcome of one document's index attempt. A provider that is not usable (none * configured, no consent, budget exhausted, cannot read the file, or errored) * is never itself a terminal outcome — the tree always falls through to the - * free local path, so the terminal reasons are only the local ones. + * free local path — but its failure NOTE survives that fall-through (see + * `IndexFailureReason`). */ export type IndexOutcome = | { indexed: true; source: ContentIndexSource; tokenCount: number } - | { - indexed: false; - reason: - "not-found" | "local-empty" | "local-unsupported" | "decrypt-error"; - }; + | { indexed: false; reason: IndexFailureReason }; + +/** + * Refs #776 — record what became of this attempt on the document row, so a + * missing index is explainable. `lastIndexAttemptAt` always advances; + * `lastIndexOutcome` carries the failure reason, or null on success (the + * `DocumentContentIndex` row itself is the success signal). Called from + * `indexLoadedDocument` (the choke point the auto job and the corpus backfill + * share) and from the manual index route. Owner-scoped; never throws — a + * diagnostic write must not fail the index work it describes. + */ +export async function recordIndexAttempt( + userId: string, + documentId: string, + outcome: IndexOutcome, +): Promise { + // No row to explain — the document is gone or not the caller's. + if (!outcome.indexed && outcome.reason === "not-found") return; + try { + await prisma.inboundDocument.updateMany({ + where: { id: documentId, userId, deletedAt: null }, + data: { + lastIndexAttemptAt: new Date(), + lastIndexOutcome: outcome.indexed ? null : outcome.reason, + }, + }); + } catch (err) { + annotate({ + action: { name: "documents.contentIndex.recordFailed" }, + meta: { documentId, reason: err instanceof Error ? err.name : "unknown" }, + }); + } +} /** * A resolved, consent-checked provider context. Resolve ONCE per user and reuse @@ -114,29 +163,56 @@ export async function resolveIndexProvider( return { chain, pick, consentOk, dailyCap }; } +/** + * The provider path's failure note when it falls through to local without a + * terminal outcome. Refs #776 — the note survives the fall-through: when the + * local path can only report "no text", the note is the recorded reason. + */ +type ProviderPathNote = + | "raster-failed" + | "pdf-needs-anthropic" + | "provider-error" + | "empty-transcription"; + +interface ProviderPathResult { + outcome: IndexOutcome | null; + note: ProviderPathNote | null; +} + /** * Try the PROVIDER (vision) path for one already-loaded document. Returns a - * terminal outcome on success/budget/provider-error, or `null` to signal - * "fall through to local" (no usable provider, or the provider cannot read this - * particular file — e.g. a PDF on a non-Anthropic account). + * terminal outcome on success / decrypt-error, or `outcome: null` to signal + * "fall through to local" (no usable provider, budget out, or the provider + * cannot read this particular file) — with a `note` naming the specific + * provider-side failure when there was one. */ async function tryProviderIndex( userId: string, document: LoadedDocument, provider: ResolvedIndexProvider, -): Promise { +): Promise { const { pick } = provider; - if (!pick || !provider.consentOk) return null; + if (!pick || !provider.consentOk) return { outcome: null, note: null }; const vision = await prepareVisionInput(document, pick.pdfSupported); if (!vision.ok) { // A decrypt failure is terminal (local would fail the same way); anything - // else (fileType / pdfNeedsAnthropic) falls through to the local path — a - // text-layer PDF on a non-Anthropic account is read locally for free. + // else falls through to the local path — a text-layer PDF on a + // non-Anthropic account is read locally for free. The PDF-specific + // failures keep their name as the note. if (vision.reason === "decryptFailed") { - return { indexed: false, reason: "decrypt-error" }; + return { + outcome: { indexed: false, reason: "decrypt-error" }, + note: null, + }; } - return null; + if (vision.reason === "rasterFailed") { + return { outcome: null, note: "raster-failed" }; + } + if (vision.reason === "pdfNeedsAnthropic") { + return { outcome: null, note: "pdf-needs-anthropic" }; + } + return { outcome: null, note: null }; } const dateKey = buildDateKey(); @@ -148,7 +224,7 @@ async function tryProviderIndex( ); // Budget exhausted → fall through to the free local path rather than stall; // a text-layer PDF stays searchable even once the AI allowance is spent. - if (!reservation.allowed) return null; + if (!reservation.allowed) return { outcome: null, note: null }; try { const { text } = await transcribeDocument({ @@ -163,6 +239,14 @@ async function tryProviderIndex( reservation.reserved, dateKey, ); + // Refs #776 — the empty-transcription guard: a provider answer with no + // text must NEVER become a "successful" empty index (it would mark the + // document indexed while making it unfindable). Fall through to local — + // a text-layer PDF still lands its text — and keep the note. The spend + // stays reconciled at the full amount: the provider was called. + if (text.trim().length === 0) { + return { outcome: null, note: "empty-transcription" }; + } const { tokenCount } = await upsertContentIndex({ userId, documentId: document.id, @@ -170,12 +254,15 @@ async function tryProviderIndex( source: "vision", providerType: pick.providerType, }); - return { indexed: true, source: "vision", tokenCount }; + return { + outcome: { indexed: true, source: "vision", tokenCount }, + note: null, + }; } catch { // Refund the reservation and let the caller fall through to local — a // transient provider miss must never leave a text-layer PDF unsearchable. await reconcileSpend(userId, reservation.reserved, 0, dateKey); - return null; + return { outcome: null, note: "provider-error" }; } } @@ -220,15 +307,38 @@ async function tryLocalIndex( /** * Index one already-loaded document: provider-first, local-fallback. Reuses a * pre-resolved provider context so a batch resolves the chain once. + * + * Refs #776 — this is the choke point the auto job and the corpus backfill + * share, so the attempt record is written HERE: every terminal outcome lands + * on the row via `recordIndexAttempt`, and a provider-path failure note + * survives a mute local fallback ("no text" is less actionable than "the + * provider errored" / "the PDF would not render"). */ export async function indexLoadedDocument( userId: string, document: LoadedDocument, provider: ResolvedIndexProvider, ): Promise { - const providerOutcome = await tryProviderIndex(userId, document, provider); - if (providerOutcome) return providerOutcome; - return tryLocalIndex(userId, document); + const providerResult = await tryProviderIndex(userId, document, provider); + let outcome: IndexOutcome; + if (providerResult.outcome) { + outcome = providerResult.outcome; + } else { + const local = await tryLocalIndex(userId, document); + if ( + !local.indexed && + providerResult.note && + (local.reason === "local-empty" || local.reason === "local-unsupported") + ) { + // The local path can only say "no text" — the provider path knows why + // the richer read never happened. Record the actionable reason. + outcome = { indexed: false, reason: providerResult.note }; + } else { + outcome = local; + } + } + await recordIndexAttempt(userId, document.id, outcome); + return outcome; } /** diff --git a/src/lib/documents/rasterize-pdf.ts b/src/lib/documents/rasterize-pdf.ts index 0545837d1..3b183fa6c 100644 --- a/src/lib/documents/rasterize-pdf.ts +++ b/src/lib/documents/rasterize-pdf.ts @@ -66,8 +66,16 @@ export interface RasterImage { dataBase64: string; } -/** Best-effort rasterization outcome — never an exception. */ -export type RasterResult = { ok: true; images: RasterImage[] } | { ok: false }; +/** + * Best-effort rasterization outcome — never an exception. Refs #776: a + * failure names its class so the caller can tell "this build cannot raster + * at all" (`unsupported` — the native canvas binary is unusable, a capability + * gap Anthropic's native PDF path would cover) from "this particular PDF + * would not render" (`render-failed` — malformed / encrypted / zero pages). + */ +export type RasterResult = + | { ok: true; images: RasterImage[] } + | { ok: false; reason: "unsupported" | "render-failed" }; // Minimal structural types for the slice of the pdfjs API this module uses. A // type-only shape (erased at build) so we never eagerly evaluate the pdfjs @@ -138,7 +146,7 @@ export async function rasterizePdf( action: { name: "documents.rasterize.failed" }, meta: { reason: "native_canvas_unsupported" }, }); - return { ok: false }; + return { ok: false, reason: "unsupported" }; } try { // Lazy import: keeps pdfjs (DOMMatrix) + @napi-rs/canvas out of the eager @@ -190,7 +198,7 @@ export async function rasterizePdf( } } - if (images.length === 0) return { ok: false }; + if (images.length === 0) return { ok: false, reason: "render-failed" }; annotate({ action: { name: "documents.rasterize.ok" }, meta: { pages: images.length, cappedFrom: doc.numPages }, @@ -207,7 +215,7 @@ export async function rasterizePdf( message: err instanceof Error ? err.message.slice(0, 300) : String(err), }, }); - return { ok: false }; + return { ok: false, reason: "render-failed" }; } finally { if (task) { try { diff --git a/src/lib/documents/store.ts b/src/lib/documents/store.ts index e2c6e0b95..15022db31 100644 --- a/src/lib/documents/store.ts +++ b/src/lib/documents/store.ts @@ -17,6 +17,7 @@ import type { InboundDocumentStatus, } from "@/generated/prisma/client"; import { servingClassFor } from "@/lib/documents/upload-policy"; +import { toIndexOutcome } from "@/lib/validations/inbound-documents"; import type { DocumentConditionLinkDto, DocumentEncounterLinkDto, @@ -225,6 +226,15 @@ export function serialiseDocument( servingClass: servingClassFor(doc.mimeType), hasContentIndex, contentIndexSource: hasContentIndex ? contentIndexSource : null, + // Refs #776 — the attempt record travels on every DTO so the detail view + // can explain a missing index. Outcome only surfaces while there IS no + // index: once indexed, the last failure is history, not status. + lastIndexAttemptAt: doc.lastIndexAttemptAt + ? doc.lastIndexAttemptAt.toISOString() + : null, + lastIndexOutcome: hasContentIndex + ? null + : toIndexOutcome(doc.lastIndexOutcome), hasThumbnail, createdAt: doc.createdAt.toISOString(), updatedAt: doc.updatedAt.toISOString(), diff --git a/src/lib/export/records-backup.ts b/src/lib/export/records-backup.ts index 34db3c428..8d0130372 100644 --- a/src/lib/export/records-backup.ts +++ b/src/lib/export/records-backup.ts @@ -133,6 +133,9 @@ export interface DocumentBackupEntry { summaryEncrypted?: string | null; summaryGeneratedAt?: string | null; summaryState?: string; + /** Refs #776 — the index-attempt record, carried verbatim (canonical DR). */ + lastIndexAttemptAt?: string | null; + lastIndexOutcome?: string | null; } export interface RecordsBackupOptions { @@ -148,6 +151,8 @@ interface DisasterRecoveryDocumentRow { summaryEncrypted: Uint8Array | null; summaryGeneratedAt: Date | null; summaryState: string; + lastIndexAttemptAt: Date | null; + lastIndexOutcome: string | null; updatedAt: Date; } @@ -254,6 +259,8 @@ export async function buildRecordsBackupSection( summaryEncrypted: true, summaryGeneratedAt: true, summaryState: true, + lastIndexAttemptAt: true, + lastIndexOutcome: true, createdAt: true, updatedAt: true, }, @@ -488,6 +495,8 @@ export async function buildRecordsBackupSection( : null, summaryGeneratedAt: dr.summaryGeneratedAt?.toISOString() ?? null, summaryState: dr.summaryState, + lastIndexAttemptAt: dr.lastIndexAttemptAt?.toISOString() ?? null, + lastIndexOutcome: dr.lastIndexOutcome, }; } diff --git a/src/lib/openapi/routes/documents.ts b/src/lib/openapi/routes/documents.ts index 2014415a7..f7337a7a2 100644 --- a/src/lib/openapi/routes/documents.ts +++ b/src/lib/openapi/routes/documents.ts @@ -171,6 +171,18 @@ const inboundDocument = z contentIndexSource: z .enum(["vision", "text-ocr", "local-pdf", "local-ocr"]) .nullable(), + lastIndexAttemptAt: z.string().nullable(), + lastIndexOutcome: z + .enum([ + "local-empty", + "local-unsupported", + "decrypt-error", + "raster-failed", + "provider-error", + "pdf-needs-anthropic", + "empty-transcription", + ]) + .nullable(), hasThumbnail: z.boolean(), createdAt: z.string(), updatedAt: z.string(), @@ -178,7 +190,7 @@ const inboundDocument = z .meta({ id: "InboundDocument", description: - "A stored document. The raw bytes are stored encrypted at rest and never returned; this is the metadata + staging summary. `status` is STORED for a freshly uploaded file (no extraction run). `title` is the user label (plaintext); `documentDate` is the user filing date; `reportDate` is the model-transcribed date (null until extraction runs). `servingClass` says how `/original` delivers the file — render inline (`inline`) or download-only (`attachment`); render/download by it, never by MIME guess. `hasContentIndex` is true when the document has a content-search index (auto-indexed on upload); `contentIndexSource` says how that index was produced — `vision` means an AI provider read the original, the other values are provider-free local extractions, and it is null when `hasContentIndex` is false. `hasThumbnail` is true when a preview thumbnail has been rendered in the background (fetch it from `/api/documents/inbound/{id}/thumbnail`); false means no preview yet or an unsupported type — show a placeholder. Treat an unknown `kind` value as OTHER when decoding.", + "A stored document. The raw bytes are stored encrypted at rest and never returned; this is the metadata + staging summary. `status` is STORED for a freshly uploaded file (no extraction run). `title` is the user label (plaintext); `documentDate` is the user filing date; `reportDate` is the model-transcribed date (null until extraction runs). `servingClass` says how `/original` delivers the file — render inline (`inline`) or download-only (`attachment`); render/download by it, never by MIME guess. `hasContentIndex` is true when the document has a content-search index (auto-indexed on upload); `contentIndexSource` says how that index was produced — `vision` means an AI provider read the original, the other values are provider-free local extractions, and it is null when `hasContentIndex` is false. `lastIndexAttemptAt` is when the most recent index attempt finished (successful or not; null when none ran); `lastIndexOutcome` says why that attempt produced no index, and is null when the attempt succeeded or none ran. `hasThumbnail` is true when a preview thumbnail has been rendered in the background (fetch it from `/api/documents/inbound/{id}/thumbnail`); false means no preview yet or an unsupported type — show a placeholder. Treat an unknown `kind` value as OTHER when decoding.", }); const inboundDocumentDetail = inboundDocument diff --git a/src/lib/validations/backup.ts b/src/lib/validations/backup.ts index b9a84977a..4b08b9ce0 100644 --- a/src/lib/validations/backup.ts +++ b/src/lib/validations/backup.ts @@ -1022,6 +1022,10 @@ const documentBackupSchema = z summaryEncrypted: base64BytesSchema.nullable().optional(), summaryGeneratedAt: isoDateTime.nullable().optional(), summaryState: z.enum(DocumentSummaryState).optional(), + // Refs #776 — the index-attempt record (canonical DR payloads only). + // Free string on purpose: a future outcome value must not fail a restore. + lastIndexAttemptAt: isoDateTime.nullable().optional(), + lastIndexOutcome: z.string().nullable().optional(), summary: z.string().nullable().optional(), createdAt: isoDateTime.optional(), updatedAt: isoDateTime.optional(), diff --git a/src/lib/validations/inbound-documents.ts b/src/lib/validations/inbound-documents.ts index 2c8c70a25..27a03ed73 100644 --- a/src/lib/validations/inbound-documents.ts +++ b/src/lib/validations/inbound-documents.ts @@ -114,6 +114,34 @@ export const DOCUMENT_SUMMARY_STATES = [ export type DocumentSummaryStateValue = (typeof DOCUMENT_SUMMARY_STATES)[number]; +/** + * Refs #776 — why a document's most recent content-index attempt produced no + * index. Mirrors the free-String `lastIndexOutcome` column (closed here, not + * in the schema, so a new reason needs no migration); the detail view renders + * a plain-words line per value instead of a bare "not indexed" that cannot + * tell "never tried" from "tried and failed". + */ +export const DOCUMENT_INDEX_OUTCOMES = [ + "local-empty", + "local-unsupported", + "decrypt-error", + "raster-failed", + "provider-error", + "pdf-needs-anthropic", + "empty-transcription", +] as const; +export type DocumentIndexOutcomeValue = + (typeof DOCUMENT_INDEX_OUTCOMES)[number]; + +/** Narrow the free-String DB column to the DTO union (null when unknown). */ +export function toIndexOutcome( + outcome: string | null | undefined, +): DocumentIndexOutcomeValue | null { + return DOCUMENT_INDEX_OUTCOMES.includes(outcome as DocumentIndexOutcomeValue) + ? (outcome as DocumentIndexOutcomeValue) + : null; +} + /** The document lifecycle states. STORED is the library default. */ export const INBOUND_DOCUMENT_STATUSES = [ "STORED", @@ -347,6 +375,18 @@ export interface InboundDocumentDto { * a locally-indexed one and offer a richer "Read with AI" pass on the latter. */ contentIndexSource: DocumentContentIndexSourceValue | null; + /** + * Refs #776 — when the most recent index attempt finished (ISO 8601), + * successful or not. Null until any attempt ran (pre-existing rows, or a + * document whose auto-index never fired). + */ + lastIndexAttemptAt: string | null; + /** + * Refs #776 — why the most recent index attempt produced no index, or null + * (attempt succeeded, or never ran — `lastIndexAttemptAt` tells the two + * apart). The detail view renders the reason in plain words. + */ + lastIndexOutcome: DocumentIndexOutcomeValue | null; /** * Whether the document has an encrypted preview thumbnail (rendered in the * background after upload). Gates the card's ``: when false the card