diff --git a/docs/api/openapi.yaml b/docs/api/openapi.yaml index 02f8a3095..780128da3 100644 --- a/docs/api/openapi.yaml +++ b/docs/api/openapi.yaml @@ -38529,13 +38529,13 @@ components: account.deletion.not_pending; activity — activity.calendar.future_date, activity.calendar.too_old; ai — ai.budget.exceeded, ai.provider.none, ai.record.notPermitted, ai.unavailable, ai.usage.aboveCeiling, ai.usage.defaultAboveCeiling, ai.usage.invalid, ai.usage.levelTooLow, ai.usage.rateLimited; ai_provider - — ai_provider.invalid, ai_provider.no_fields; aiRuns — aiRuns.failed, aiRuns.notFound, aiRuns.timedOut, - aiRuns.workerUnavailable; allergy — allergy.invalid; analytics — analytics.invalid_query; anamnesis — - anamnesis.auditC.invalid, anamnesis.auditC.notFound, anamnesis.auditC.rateLimited, - anamnesis.fact.conflict, anamnesis.fact.currentExists, anamnesis.fact.invalidValue, - anamnesis.lifeConfirm.invalid, anamnesis.lifeSections.invalid; auth — auth.account.deletion_pending, - auth.admin.required, auth.already_authenticated, auth.mfa.code_invalid, auth.missing, - auth.refresh.invalid, auth.refresh.reuse, auth.refresh.revoked, auth.reproof.failed, + — ai_provider.invalid, ai_provider.no_fields; aiRuns — aiRuns.failed, aiRuns.notFound, + aiRuns.readerBusy, aiRuns.timedOut, aiRuns.workerUnavailable; allergy — allergy.invalid; analytics — + analytics.invalid_query; anamnesis — anamnesis.auditC.invalid, anamnesis.auditC.notFound, + anamnesis.auditC.rateLimited, anamnesis.fact.conflict, anamnesis.fact.currentExists, + anamnesis.fact.invalidValue, anamnesis.lifeConfirm.invalid, anamnesis.lifeSections.invalid; auth — + auth.account.deletion_pending, auth.admin.required, auth.already_authenticated, auth.mfa.code_invalid, + auth.missing, auth.refresh.invalid, auth.refresh.reuse, auth.refresh.revoked, auth.reproof.failed, auth.reproof.required, auth.reproof.too_weak, auth.scope.insufficient, auth.session.insecure_transport, auth.session.unavailable, auth.stepup.required, auth.token.expired, auth.token.invalid; backup — backup.foreign_reference, backup.key.missing, backup.payload.undecryptable, backup.restore.active, @@ -75840,7 +75840,8 @@ components: description: "Present once `status` is `FAILED`: the `meta.errorCode`, message and HTTP status the queuing route would have answered with, or one of the run's own codes: `aiRuns.workerUnavailable` (no worker took the run within fifteen minutes), `aiRuns.timedOut` (the read outlived the calls the AI response time allows), - `aiRuns.failed` (anything else)." + `aiRuns.readerBusy` (the server's document reader was busy past the wait a read gets, or stopped before it + answered; nothing was read or charged, try again in a minute), `aiRuns.failed` (anything else)." createdAt: type: string startedAt: diff --git a/messages/de.json b/messages/de.json index 96a78eff5..1ee556a07 100644 --- a/messages/de.json +++ b/messages/de.json @@ -175,6 +175,9 @@ "failed": "Das Etikett konnte nicht gelesen werden. Versuch ein schärferes Foto." } } + }, + "aiRuns": { + "readerBusy": "Der Dokumentleser ist gerade mit anderen Aufgaben beschäftigt. Versuch es in einer Minute noch einmal." } }, "doctorReport": { @@ -13258,7 +13261,8 @@ "backgroundScan": "Wird im Hintergrund gelesen. Mit einem langsamen Modell kann das ein paar Minuten dauern.", "waitingForWorker": "Warte auf den Hintergrunddienst. Ändert sich das nicht, läuft der Worker auf dem Server vielleicht nicht.", "errorWorkerUnavailable": "Der Hintergrunddienst ist gerade nicht erreichbar. Versuch es gleich noch einmal oder bitte die Person, die den Server betreibt, ihn zu prüfen.", - "errorTimedOut": "Das Lesen hat länger gedauert, als deine KI-Antwortzeit erlaubt. Versuch es noch einmal oder erhöhe die Antwortzeit in den KI-Einstellungen." + "errorTimedOut": "Das Lesen hat länger gedauert, als deine KI-Antwortzeit erlaubt. Versuch es noch einmal oder erhöhe die Antwortzeit in den KI-Einstellungen.", + "errorReaderBusy": "Der Dokumentleser ist gerade mit anderen Aufgaben beschäftigt. Versuch es in einer Minute noch einmal." }, "symptoms": { "add": "Neues Symptom", diff --git a/messages/en.json b/messages/en.json index bb4cf34c4..c3ab4fd71 100644 --- a/messages/en.json +++ b/messages/en.json @@ -175,6 +175,9 @@ "failed": "The label could not be read. Try a sharper photo." } } + }, + "aiRuns": { + "readerBusy": "The document reader is busy with other work. Try again in a minute." } }, "doctorReport": { @@ -13258,7 +13261,8 @@ "backgroundScan": "Reading in the background. With a slow model this can take a few minutes.", "waitingForWorker": "Waiting for the background worker. If this does not change, the server's worker may not be running.", "errorWorkerUnavailable": "The background worker is not available right now. Try again in a moment, or ask the server operator to check it.", - "errorTimedOut": "Reading took longer than your AI response time allows. Try again, or raise the response time in the AI settings." + "errorTimedOut": "Reading took longer than your AI response time allows. Try again, or raise the response time in the AI settings.", + "errorReaderBusy": "The document reader is busy with other work. Try again in a minute." }, "symptoms": { "add": "New symptom", diff --git a/messages/es.json b/messages/es.json index 5c9056ae2..11c5c0dfd 100644 --- a/messages/es.json +++ b/messages/es.json @@ -175,6 +175,9 @@ "failed": "No se pudo leer la etiqueta. Prueba con una foto más nítida." } } + }, + "aiRuns": { + "readerBusy": "El lector de documentos está ocupado con otras tareas. Inténtalo de nuevo en un minuto." } }, "doctorReport": { @@ -13258,7 +13261,8 @@ "backgroundScan": "Leyendo en segundo plano. Con un modelo lento puede tardar unos minutos.", "waitingForWorker": "Esperando al proceso en segundo plano. Si no cambia, puede que el worker del servidor no esté en marcha.", "errorWorkerUnavailable": "El proceso en segundo plano no está disponible ahora mismo. Inténtalo de nuevo en un momento o pide a quien administra el servidor que lo revise.", - "errorTimedOut": "La lectura tardó más de lo que permite tu tiempo de respuesta de IA. Inténtalo de nuevo o aumenta el tiempo de respuesta en los ajustes de IA." + "errorTimedOut": "La lectura tardó más de lo que permite tu tiempo de respuesta de IA. Inténtalo de nuevo o aumenta el tiempo de respuesta en los ajustes de IA.", + "errorReaderBusy": "El lector de documentos está ocupado con otras tareas. Inténtalo de nuevo en un minuto." }, "symptoms": { "add": "Nuevo síntoma", diff --git a/messages/fr.json b/messages/fr.json index ddc754baf..865cc6f52 100644 --- a/messages/fr.json +++ b/messages/fr.json @@ -175,6 +175,9 @@ "failed": "L’étiquette n’a pas pu être lue. Essayez une photo plus nette." } } + }, + "aiRuns": { + "readerBusy": "Le lecteur de documents est occupé par d'autres tâches. Réessayez dans une minute." } }, "doctorReport": { @@ -13258,7 +13261,8 @@ "backgroundScan": "Lecture en arrière-plan. Avec un modèle lent, cela peut prendre quelques minutes.", "waitingForWorker": "En attente du traitement en arrière-plan. Si rien ne change, le worker du serveur n'est peut-être pas lancé.", "errorWorkerUnavailable": "Le traitement en arrière-plan n'est pas disponible pour le moment. Réessayez dans un instant ou demandez à la personne qui gère le serveur de le vérifier.", - "errorTimedOut": "La lecture a pris plus de temps que votre délai de réponse IA ne le permet. Réessayez ou augmentez ce délai dans les réglages IA." + "errorTimedOut": "La lecture a pris plus de temps que votre délai de réponse IA ne le permet. Réessayez ou augmentez ce délai dans les réglages IA.", + "errorReaderBusy": "Le lecteur de documents est occupé par d'autres tâches. Réessayez dans une minute." }, "symptoms": { "add": "Nouveau symptôme", diff --git a/messages/it.json b/messages/it.json index fe6450c8d..f7daf94b8 100644 --- a/messages/it.json +++ b/messages/it.json @@ -175,6 +175,9 @@ "failed": "Non è stato possibile leggere l'etichetta. Prova con una foto più nitida." } } + }, + "aiRuns": { + "readerBusy": "Il lettore di documenti è occupato con altre attività. Riprova tra un minuto." } }, "doctorReport": { @@ -13258,7 +13261,8 @@ "backgroundScan": "Lettura in background. Con un modello lento può richiedere qualche minuto.", "waitingForWorker": "In attesa del processo in background. Se non cambia, il worker del server potrebbe non essere attivo.", "errorWorkerUnavailable": "Il processo in background non è disponibile al momento. Riprova tra poco o chiedi a chi gestisce il server di controllarlo.", - "errorTimedOut": "La lettura ha richiesto più tempo di quanto consenta il tuo tempo di risposta dell'IA. Riprova o aumenta il tempo di risposta nelle impostazioni IA." + "errorTimedOut": "La lettura ha richiesto più tempo di quanto consenta il tuo tempo di risposta dell'IA. Riprova o aumenta il tempo di risposta nelle impostazioni IA.", + "errorReaderBusy": "Il lettore di documenti è occupato con altre attività. Riprova tra un minuto." }, "symptoms": { "add": "Nuovo sintomo", diff --git a/messages/ko.json b/messages/ko.json index 2ef2164d3..f702d430c 100644 --- a/messages/ko.json +++ b/messages/ko.json @@ -175,6 +175,9 @@ "failed": "라벨을 읽지 못했어요. 더 선명한 사진으로 다시 시도하세요." } } + }, + "aiRuns": { + "readerBusy": "문서 판독기가 다른 작업으로 바빠요. 1분 후에 다시 시도하세요." } }, "doctorReport": { @@ -13258,7 +13261,8 @@ "backgroundScan": "백그라운드에서 읽고 있어요. 느린 모델이면 몇 분 걸릴 수 있어요.", "waitingForWorker": "백그라운드 작업자를 기다리고 있어요. 계속 그대로라면 서버의 작업자가 실행 중이 아닐 수 있어요.", "errorWorkerUnavailable": "지금은 백그라운드 작업자를 사용할 수 없어요. 잠시 후 다시 시도하거나 서버 운영자에게 확인을 요청하세요.", - "errorTimedOut": "AI 응답 시간 설정보다 읽기가 오래 걸렸어요. 다시 시도하거나 AI 설정에서 응답 시간을 늘려 주세요." + "errorTimedOut": "AI 응답 시간 설정보다 읽기가 오래 걸렸어요. 다시 시도하거나 AI 설정에서 응답 시간을 늘려 주세요.", + "errorReaderBusy": "문서 판독기가 다른 작업으로 바빠요. 1분 후에 다시 시도하세요." }, "symptoms": { "add": "새 증상", diff --git a/messages/nl.json b/messages/nl.json index 9bd5560ec..33897f904 100644 --- a/messages/nl.json +++ b/messages/nl.json @@ -175,6 +175,9 @@ "failed": "Het etiket kon niet worden gelezen. Probeer een scherpere foto." } } + }, + "aiRuns": { + "readerBusy": "De documentlezer is met ander werk bezig. Probeer het over een minuut opnieuw." } }, "doctorReport": { @@ -13258,7 +13261,8 @@ "backgroundScan": "Wordt op de achtergrond gelezen. Met een traag model kan dit een paar minuten duren.", "waitingForWorker": "Wachten op het achtergrondproces. Als dit niet verandert, draait het achtergrondproces van de server mogelijk niet.", "errorWorkerUnavailable": "Het achtergrondproces is op dit moment niet beschikbaar. Probeer het zo opnieuw, of vraag de beheerder van de server om het te controleren.", - "errorTimedOut": "Het lezen duurde langer dan je ingestelde AI-responstijd toestaat. Probeer het opnieuw, of verhoog de responstijd in de AI-instellingen." + "errorTimedOut": "Het lezen duurde langer dan je ingestelde AI-responstijd toestaat. Probeer het opnieuw, of verhoog de responstijd in de AI-instellingen.", + "errorReaderBusy": "De documentlezer is met ander werk bezig. Probeer het over een minuut opnieuw." }, "symptoms": { "add": "Nieuw symptoom", diff --git a/messages/pl.json b/messages/pl.json index 2a1673bcd..1c33d3938 100644 --- a/messages/pl.json +++ b/messages/pl.json @@ -175,6 +175,9 @@ "failed": "Nie udało się odczytać etykiety. Spróbuj ostrzejszego zdjęcia." } } + }, + "aiRuns": { + "readerBusy": "Czytnik dokumentów jest zajęty innymi zadaniami. Spróbuj ponownie za minutę." } }, "doctorReport": { @@ -13258,7 +13261,8 @@ "backgroundScan": "Odczyt w tle. Przy wolnym modelu może to potrwać kilka minut.", "waitingForWorker": "Oczekiwanie na proces w tle. Jeśli nic się nie zmienia, worker na serwerze może nie działać.", "errorWorkerUnavailable": "Proces w tle jest teraz niedostępny. Spróbuj ponownie za chwilę albo poproś administratora serwera o sprawdzenie go.", - "errorTimedOut": "Odczyt trwał dłużej, niż pozwala ustawiony czas odpowiedzi AI. Spróbuj ponownie albo wydłuż czas odpowiedzi w ustawieniach AI." + "errorTimedOut": "Odczyt trwał dłużej, niż pozwala ustawiony czas odpowiedzi AI. Spróbuj ponownie albo wydłuż czas odpowiedzi w ustawieniach AI.", + "errorReaderBusy": "Czytnik dokumentów jest zajęty innymi zadaniami. Spróbuj ponownie za minutę." }, "symptoms": { "add": "Nowy objaw", diff --git a/messages/pt-BR.json b/messages/pt-BR.json index cc011e70a..60606bfc6 100644 --- a/messages/pt-BR.json +++ b/messages/pt-BR.json @@ -175,6 +175,9 @@ "failed": "Não foi possível ler o rótulo. Tente uma foto mais nítida." } } + }, + "aiRuns": { + "readerBusy": "O leitor de documentos está ocupado com outras tarefas. Tente novamente em um minuto." } }, "doctorReport": { @@ -13258,7 +13261,8 @@ "backgroundScan": "Lendo em segundo plano. Com um modelo lento, isso pode levar alguns minutos.", "waitingForWorker": "Aguardando o worker em segundo plano. Se isso não mudar, o worker do servidor pode não estar em execução.", "errorWorkerUnavailable": "O worker em segundo plano não está disponível agora. Tente novamente em instantes, ou peça ao operador do servidor para verificá-lo.", - "errorTimedOut": "A leitura demorou mais do que o seu tempo de resposta de IA permite. Tente novamente, ou aumente o tempo de resposta nas configurações de IA." + "errorTimedOut": "A leitura demorou mais do que o seu tempo de resposta de IA permite. Tente novamente, ou aumente o tempo de resposta nas configurações de IA.", + "errorReaderBusy": "O leitor de documentos está ocupado com outras tarefas. Tente novamente em um minuto." }, "symptoms": { "add": "Novo sintoma", diff --git a/src/components/documents/use-document-assist.ts b/src/components/documents/use-document-assist.ts index c47fb646f..999590bca 100644 --- a/src/components/documents/use-document-assist.ts +++ b/src/components/documents/use-document-assist.ts @@ -228,6 +228,8 @@ export function documentAiErrorKey(err: unknown): string { return "aiRuns.errorWorkerUnavailable"; case "aiRuns.timedOut": return "aiRuns.errorTimedOut"; + case "aiRuns.readerBusy": + return "aiRuns.errorReaderBusy"; case "consent.ai.required": return "documents.assist.errorConsent"; case "ai.provider.none": diff --git a/src/components/labs/ocr-review-dialog.tsx b/src/components/labs/ocr-review-dialog.tsx index e98317b24..f2310f561 100644 --- a/src/components/labs/ocr-review-dialog.tsx +++ b/src/components/labs/ocr-review-dialog.tsx @@ -117,6 +117,8 @@ export function extractErrorMessage( return t("aiRuns.errorWorkerUnavailable"); case "aiRuns.timedOut": return t("aiRuns.errorTimedOut"); + case "aiRuns.readerBusy": + return t("aiRuns.errorReaderBusy"); default: break; } diff --git a/src/lib/documents/ai-route-support.ts b/src/lib/documents/ai-route-support.ts index 5a903e484..9f89710f8 100644 --- a/src/lib/documents/ai-route-support.ts +++ b/src/lib/documents/ai-route-support.ts @@ -154,9 +154,14 @@ export type VisionInput = // Refs #776 — `rasterFailed` (this PDF would not render on a raster- // capable build) is distinct from `pdfNeedsAnthropic` (this build / // provider cannot take a PDF at all): they need different fixes, so - // they must not share a label. + // they must not share a label. `readerBusy` is neither: the reader + // could not take the document now, and the work is tried again. reason: - "decryptFailed" | "fileType" | "pdfNeedsAnthropic" | "rasterFailed"; + | "decryptFailed" + | "fileType" + | "pdfNeedsAnthropic" + | "rasterFailed" + | "readerBusy"; }; /** @@ -211,6 +216,9 @@ export async function prepareVisionInput( if (raster.ok) { return { ok: true, images: raster.images, documents: [] }; } + if (raster.reason === "busy" || raster.reason === "interrupted") { + return { ok: false, reason: "readerBusy" }; + } // Refs #776 — a build that cannot rasterize at all is a capability gap a // PDF-native (Anthropic) provider would cover; a render failure on a // capable build is a property of THIS document. Report them apart. diff --git a/src/lib/documents/ai-runs/extract-run.ts b/src/lib/documents/ai-runs/extract-run.ts index c968e0d54..3d34a1768 100644 --- a/src/lib/documents/ai-runs/extract-run.ts +++ b/src/lib/documents/ai-runs/extract-run.ts @@ -52,6 +52,7 @@ import { dateOnlyAtNoonUtc } from "@/lib/tz/date-only"; import { refusalAsFailure } from "./http"; import { meteredReadBudget } from "./metered-budget"; import { + readerBusyFailure, runFailure, unreadableDocument, type DocumentTextPick, @@ -306,7 +307,9 @@ export async function executeDocumentExtract(args: { source = { mode: "text", text: input.text }; } const plan = await planInboundReads(source, schedule); - if (!plan.ok) return refused(pdfUnreadable()); + if (!plan.ok) { + return refused(plan.busy ? readerBusyFailure() : pdfUnreadable()); + } await schedule.arm(plan.args.chunks.length * OCR_EXTRACT_MODEL_CALLS); let result: InboundExtractionResult; diff --git a/src/lib/documents/ai-runs/index-run.ts b/src/lib/documents/ai-runs/index-run.ts index 64cf2251c..c04c6ec43 100644 --- a/src/lib/documents/ai-runs/index-run.ts +++ b/src/lib/documents/ai-runs/index-run.ts @@ -32,6 +32,7 @@ import { recordIndexAttempt } from "@/lib/documents/index-document"; import type { requireDocumentVisionProvider } from "@/lib/documents/provider-order"; import { annotate } from "@/lib/logging/context"; +import { readerBusyFailure } from "./run-support"; import { settleAiRunBudget } from "./store"; import type { AiRunBudget, @@ -157,6 +158,8 @@ export async function executeDocumentIndex(args: { // gone. await settleAiRunBudget(userId, budget, 0, null); await refundDocumentAiSlot(userId); + // A busy reader says nothing about the document: no attempt is recorded. + if (vision.reason === "readerBusy") return readerBusyFailure(); if (vision.reason === "pdfNeedsAnthropic") { await recordIndexAttempt(userId, document.id, { indexed: false, diff --git a/src/lib/documents/ai-runs/run-support.ts b/src/lib/documents/ai-runs/run-support.ts index 340f86b10..0aa291830 100644 --- a/src/lib/documents/ai-runs/run-support.ts +++ b/src/lib/documents/ai-runs/run-support.ts @@ -13,7 +13,7 @@ import type { } from "@/lib/documents/provider-order"; import type { VisionInput } from "@/lib/documents/ai-route-support"; -import type { AiRunOutcome } from "./types"; +import { AI_RUN_ERROR_CODES, type AiRunOutcome } from "./types"; export type DocumentVisionPick = Awaited< ReturnType @@ -44,6 +44,19 @@ export function unreadableDocument(parseError: boolean): AiRunOutcome { }); } +/** + * The document reader was busy with other work for longer than a person's + * read waits, or its process ended for no reason it measured. Nothing was + * read and nothing is charged; asking again in a minute is the answer. + */ +export function readerBusyFailure(): AiRunOutcome { + return runFailure({ + status: 503, + message: "The document reader is busy. Try again in a minute.", + errorCode: AI_RUN_ERROR_CODES.readerBusy, + }); +} + /** * A stored original that could not become a provider input, as the summary * and suggest routes have always answered it. @@ -59,6 +72,7 @@ export function visionPreparationFailure( errorCode: "documents.inbound.pdfNeedsAnthropic", }); } + if (reason === "readerBusy") return readerBusyFailure(); if (reason === "fileType") { return runFailure({ status: 422, diff --git a/src/lib/documents/ai-runs/types.ts b/src/lib/documents/ai-runs/types.ts index 69d22ddd6..79155281a 100644 --- a/src/lib/documents/ai-runs/types.ts +++ b/src/lib/documents/ai-runs/types.ts @@ -74,6 +74,7 @@ export const AI_RUN_ERROR_CODES = { workerUnavailable: "aiRuns.workerUnavailable", timedOut: "aiRuns.timedOut", failed: "aiRuns.failed", + readerBusy: "aiRuns.readerBusy", } as const; /** How long a queued run waits for a worker before it is failed. */ diff --git a/src/lib/documents/extract.ts b/src/lib/documents/extract.ts index 361faa61a..a1660adf5 100644 --- a/src/lib/documents/extract.ts +++ b/src/lib/documents/extract.ts @@ -484,7 +484,7 @@ export function planInboundTextReads( export async function planInboundReads( source: InboundReadSource, schedule: OcrSchedule, -): Promise<({ ok: true } & InboundReads) | { ok: false }> { +): Promise<({ ok: true } & InboundReads) | { ok: false; busy?: true }> { if (source.mode === "text") { return { ok: true, ...planInboundTextReads(source.text, schedule) }; } @@ -494,7 +494,7 @@ export async function planInboundReads( source.pdfSupported, schedule, ); - if (!plan.ok) return { ok: false }; + if (!plan.ok) return plan; return { ok: true, args: { mode: "vision", chunks: plan.chunks, totalPages: plan.totalPages }, diff --git a/src/lib/documents/index-document.ts b/src/lib/documents/index-document.ts index 69ae2ff41..cd9f6e259 100644 --- a/src/lib/documents/index-document.ts +++ b/src/lib/documents/index-document.ts @@ -58,6 +58,7 @@ import { import { transcribeDocument } from "@/lib/documents/describe"; import { documentAutoReadEnabled } from "@/lib/documents/document-settings"; import { localExtractText } from "@/lib/documents/local-extract"; +import { PdfReaderUnavailableError } from "@/lib/documents/pdfjs"; import { resolveDocumentVisionProvider } from "@/lib/documents/provider-order"; import { decryptDocumentContent } from "@/lib/documents/store"; import { prisma } from "@/lib/db"; @@ -238,6 +239,12 @@ async function tryProviderIndex( if (vision.reason === "rasterFailed") { return { outcome: null, note: "raster-failed" }; } + if (vision.reason === "readerBusy") { + // The reader could not take the document now. Falling back to the + // local text layer would settle for the lesser read for good; the job + // runs again instead. + throw new PdfReaderUnavailableError("busy"); + } if (vision.reason === "pdfNeedsAnthropic") { return { outcome: null, note: "pdf-needs-anthropic" }; } diff --git a/src/lib/documents/ocr-plan.ts b/src/lib/documents/ocr-plan.ts index f46242133..b23a108f1 100644 --- a/src/lib/documents/ocr-plan.ts +++ b/src/lib/documents/ocr-plan.ts @@ -170,7 +170,12 @@ export type OcrVisionPlan = /** Set when the PDF is longer than the page cap: only the first pages are read. */ pageCoverage?: { read: number; total: number }; } - | { ok: false }; + /** + * `busy`: the reader could not take the PDF now (busy past the wait a read + * gets, or its process ended for no measured reason). Not a verdict on the + * file; the person asks again. + */ + | { ok: false; busy?: true }; /** * Plan the reads of an image or a PDF. Every PDF is rendered first: a report @@ -226,7 +231,12 @@ export async function planVisionReads( : {}), }; } - if (!pdfSupported) return { ok: false }; + if (!pdfSupported) { + return !raster.ok && + (raster.reason === "busy" || raster.reason === "interrupted") + ? { ok: false, busy: true } + : { ok: false }; + } return { ok: true, chunks: [ diff --git a/src/lib/jobs/__tests__/document-ai-run.test.ts b/src/lib/jobs/__tests__/document-ai-run.test.ts index a4cd54d32..7de034a2d 100644 --- a/src/lib/jobs/__tests__/document-ai-run.test.ts +++ b/src/lib/jobs/__tests__/document-ai-run.test.ts @@ -285,6 +285,32 @@ describe("runDocumentAiRun", () => { ); }); + it("reads as interactive work of the run's owner, so the document reader serves it first", async () => { + let seen: unknown = null; + executeDocumentIndex.mockImplementation(async () => { + const { currentPdfWork } = await import("@/lib/documents/pdf-isolation"); + seen = currentPdfWork(); + return { ok: true, data: {} }; + }); + await runDocumentAiRun("run1", NOW + 2_700_000, now); + expect(seen).toEqual({ owner: "u1", lane: "interactive" }); + }); + + it("answers a busy document reader with a retryable code and hands the reservation back", async () => { + const { PdfReaderUnavailableError } = await import("@/lib/documents/pdfjs"); + executeDocumentIndex.mockRejectedValue( + new PdfReaderUnavailableError("busy"), + ); + expect(await runDocumentAiRun("run1", NOW + 2_700_000, now)).toBe("failed"); + expect(settleAiRunBudget).toHaveBeenCalledWith("u1", budget, 0, null); + expect(failAiRun).toHaveBeenCalledWith( + "run1", + expect.objectContaining({ status: 503, errorCode: "aiRuns.readerBusy" }), + "RUNNING", + new Date(NOW), + ); + }); + it("fails a run whose document is gone and hands the reservation back", async () => { loadOwnedDocument.mockResolvedValue(null); expect(await runDocumentAiRun("run1", NOW + 2_700_000, now)).toBe("failed"); diff --git a/src/lib/jobs/document-ai-run.ts b/src/lib/jobs/document-ai-run.ts index 4df270a6a..69949a2b0 100644 --- a/src/lib/jobs/document-ai-run.ts +++ b/src/lib/jobs/document-ai-run.ts @@ -42,6 +42,8 @@ import { loadOwnedDocument } from "@/lib/documents/ai-route-support"; import { loadDocumentChatText } from "@/lib/documents/content-index"; import { refusalAsFailure } from "@/lib/documents/ai-runs/http"; import { executeDocumentExtract } from "@/lib/documents/ai-runs/extract-run"; +import { withPdfWork } from "@/lib/documents/pdf-isolation"; +import { PdfReaderUnavailableError } from "@/lib/documents/pdfjs"; import { DOCUMENT_INDEX_MODEL_CALLS, executeDocumentIndex, @@ -614,7 +616,12 @@ export async function runDocumentAiRun( let outcome: AiRunOutcome; try { - outcome = await RUN_DISPATCH[run.kind](run, deadlineAt, now); + // A person asked for this read and is waiting on it: the document + // reader serves it ahead of background work, in turn with other people. + outcome = await withPdfWork( + { owner: run.userId, lane: "interactive" }, + () => RUN_DISPATCH[run.kind](run, deadlineAt, now), + ); } catch (err) { // Everything that throws here threw before the body took over the // reservation, so it goes back here. @@ -622,6 +629,13 @@ export async function runDocumentAiRun( const failure: AiRunFailure = (err instanceof RunRefused ? err.failure : null) ?? + (err instanceof PdfReaderUnavailableError + ? { + status: 503, + errorCode: AI_RUN_ERROR_CODES.readerBusy, + message: "The document reader is busy. Try again in a minute.", + } + : null) ?? refusalAsFailure(err) ?? { status: 500, errorCode: AI_RUN_ERROR_CODES.failed, diff --git a/src/lib/jobs/document-content-index-backfill.ts b/src/lib/jobs/document-content-index-backfill.ts index 45fc3d078..35a062761 100644 --- a/src/lib/jobs/document-content-index-backfill.ts +++ b/src/lib/jobs/document-content-index-backfill.ts @@ -55,6 +55,7 @@ import { upsertContentIndex, } from "@/lib/documents/content-index"; import { transcribeDocument } from "@/lib/documents/describe"; +import { withPdfWork } from "@/lib/documents/pdf-isolation"; import { getGlobalBoss } from "@/lib/jobs/boss-instance"; import { resolveDocumentVisionProvider } from "@/lib/documents/provider-order"; import { annotate } from "@/lib/logging/context"; @@ -320,7 +321,10 @@ export async function runContentIndexBackfillForUser( skipped += 1; continue; } - const vision = await prepareVisionInput(document, pick.pdfSupported); + const vision = await withPdfWork( + { owner: userId, lane: "background" }, + () => prepareVisionInput(document, pick.pdfSupported), + ); if (!vision.ok) { // Not vision-indexable after all (undecryptable, unreadable bytes, // or a PDF the rasteriser could not render) — refund and move on so diff --git a/src/lib/jobs/document-index.ts b/src/lib/jobs/document-index.ts index 710e3552c..ca5d027e9 100644 --- a/src/lib/jobs/document-index.ts +++ b/src/lib/jobs/document-index.ts @@ -15,6 +15,7 @@ import { STATUS_MODEL_CALLS } from "@/lib/ai/ai-budgets"; import { maybeAutoStageLabFacts } from "@/lib/documents/auto-stage-labs"; import { indexDocumentContent } from "@/lib/documents/index-document"; +import { withPdfWork } from "@/lib/documents/pdf-isolation"; import type { OcrSchedule } from "@/lib/documents/ocr-plan"; import { ocrReadCallMs } from "@/lib/documents/ocr-read"; import { prisma } from "@/lib/db"; @@ -50,9 +51,9 @@ export async function runDocumentIndex( const { userId, documentId } = payload; if (!userId || !documentId) return; const localOnly = payload.localOnly === true; - const outcome = await indexDocumentContent(userId, documentId, { - localOnly, - }); + const outcome = await withPdfWork({ owner: userId, lane: "background" }, () => + indexDocumentContent(userId, documentId, { localOnly }), + ); annotate({ action: { name: "documents.autoIndex.run" }, meta: outcome.indexed @@ -72,9 +73,9 @@ export async function runDocumentIndex( options.deadlineAt === undefined ? undefined : await stagingSchedule(userId, options.deadlineAt, options.now); - await maybeAutoStageLabFacts(userId, documentId, { schedule }).catch( - caughtAs("documents.auto_index.lab_staging_failed"), - ); + await withPdfWork({ owner: userId, lane: "background" }, () => + maybeAutoStageLabFacts(userId, documentId, { schedule }), + ).catch(caughtAs("documents.auto_index.lab_staging_failed")); } } diff --git a/src/lib/jobs/document-summary.ts b/src/lib/jobs/document-summary.ts index a1a9bbb84..9b421da34 100644 --- a/src/lib/jobs/document-summary.ts +++ b/src/lib/jobs/document-summary.ts @@ -43,6 +43,8 @@ import type { Locale } from "@/lib/i18n/config"; import { resolveJobLocale } from "@/lib/i18n/job-locale"; import { documentAutoReadEnabled } from "@/lib/documents/document-settings"; import { resolveDocumentVisionProvider } from "@/lib/documents/provider-order"; +import { withPdfWork } from "@/lib/documents/pdf-isolation"; +import { PdfReaderUnavailableError } from "@/lib/documents/pdfjs"; import { encryptDocumentSummary } from "@/lib/documents/store"; import { prisma } from "@/lib/db"; import { getGlobalBoss } from "@/lib/jobs/boss-instance"; @@ -81,6 +83,25 @@ export interface DocumentSummaryPayload { enqueuedAt?: string; } +/** + * The stored original as provider input, read as this account's background + * work. A busy reader is not a property of the document: the job throws, and + * pg-boss runs it again. + */ +async function prepareSummaryVision( + userId: string, + document: Parameters[0], + pick: { pdfSupported: boolean }, +): Promise> { + const vision = await withPdfWork({ owner: userId, lane: "background" }, () => + prepareVisionInput(document, pick.pdfSupported), + ); + if (!vision.ok && vision.reason === "readerBusy") { + throw new PdfReaderUnavailableError("busy"); + } + return vision; +} + /** * Generate + persist ONE document's background summary (owner-scoped). Never * throws for an expected outcome — every gate that fails resolves to a tagged @@ -210,7 +231,7 @@ export async function runDocumentSummaryJob( const document = await loadOwnedDocument(userId, documentId); if (!document) return; - const vision = await prepareVisionInput(document, pick.pdfSupported); + const vision = await prepareSummaryVision(userId, document, pick); if (!vision.ok) { // Cannot read this file with the picked provider (decrypt miss, unsupported // type, or a PDF needing an Anthropic provider). Graceful no-op. diff --git a/src/lib/labs/ocr-run.ts b/src/lib/labs/ocr-run.ts index 75b668be5..b122cd55b 100644 --- a/src/lib/labs/ocr-run.ts +++ b/src/lib/labs/ocr-run.ts @@ -20,6 +20,7 @@ import { AI_BUDGETS } from "@/lib/ai/ai-budgets"; import { refusalAsFailure } from "@/lib/documents/ai-runs/http"; import type { ProviderChainType } from "@/lib/ai/provider-chain"; import { meteredReadBudget } from "@/lib/documents/ai-runs/metered-budget"; +import { readerBusyFailure } from "@/lib/documents/ai-runs/run-support"; import { AI_RUN_ERROR_CODES, type AiRunBudget, @@ -221,6 +222,10 @@ export async function executeOcrExtraction(args: { input.pick.pdfSupported, schedule, ); + if (!plan.ok && plan.busy) { + await settle(false); + return readerBusyFailure(); + } if (!plan.ok) { annotate({ action: { name: "labs.ocr.fileRejected" }, diff --git a/src/lib/medications/ingredients/label-extract.ts b/src/lib/medications/ingredients/label-extract.ts index 14430b10c..2379f5afd 100644 --- a/src/lib/medications/ingredients/label-extract.ts +++ b/src/lib/medications/ingredients/label-extract.ts @@ -29,6 +29,7 @@ import { type AIProvider, type CompletionParams, } from "@/lib/ai/types"; +import { readerBusyFailure } from "@/lib/documents/ai-runs/run-support"; import { settleAiRunBudget } from "@/lib/documents/ai-runs/store"; import type { AiRunBudget, @@ -232,6 +233,13 @@ export async function executeSupplementLabelExtraction(args: { ]; } else { const raster = await rasterizePdf(bytes); + if ( + !raster.ok && + (raster.reason === "busy" || raster.reason === "interrupted") + ) { + await settleAiRunBudget(userId, budget, 0, null); + return readerBusyFailure(); + } if (!raster.ok) { await settleAiRunBudget(userId, budget, 0, null); return failure({ diff --git a/src/lib/openapi/error-codes.ts b/src/lib/openapi/error-codes.ts index 2b49765fd..a36ad0286 100644 --- a/src/lib/openapi/error-codes.ts +++ b/src/lib/openapi/error-codes.ts @@ -57,6 +57,7 @@ export const ERROR_CODE_CATALOGUE: Readonly> = aiRuns: [ "aiRuns.failed", "aiRuns.notFound", + "aiRuns.readerBusy", "aiRuns.timedOut", "aiRuns.workerUnavailable", ], diff --git a/src/lib/openapi/routes/ai-runs.ts b/src/lib/openapi/routes/ai-runs.ts index 1be07ade9..8be799378 100644 --- a/src/lib/openapi/routes/ai-runs.ts +++ b/src/lib/openapi/routes/ai-runs.ts @@ -64,7 +64,7 @@ const aiRun = z }) .nullable() .describe( - "Present once `status` is `FAILED`: the `meta.errorCode`, message and HTTP status the queuing route would have answered with, or one of the run's own codes: `aiRuns.workerUnavailable` (no worker took the run within fifteen minutes), `aiRuns.timedOut` (the read outlived the calls the AI response time allows), `aiRuns.failed` (anything else).", + "Present once `status` is `FAILED`: the `meta.errorCode`, message and HTTP status the queuing route would have answered with, or one of the run's own codes: `aiRuns.workerUnavailable` (no worker took the run within fifteen minutes), `aiRuns.timedOut` (the read outlived the calls the AI response time allows), `aiRuns.readerBusy` (the server's document reader was busy past the wait a read gets, or stopped before it answered; nothing was read or charged, try again in a minute), `aiRuns.failed` (anything else).", ), createdAt: z.string(), startedAt: z.string().nullable(),