fix(documents): tell people when the document reader is busy, and attribute its work

A read a person started now runs as interactive work of their account, and
the document jobs as background work of theirs, so the reader's queue can
order and cap them. When the reader is busy or stopped before it answered,
a lab scan, label read or document read fails with aiRuns.readerBusy (503,
nothing read or charged) instead of a misleading unreadable-file error, and
the index and summary jobs throw so the queue runs them again rather than
recording an outcome about the document.
This commit is contained in:
Marc-André Bombeck committed 2026-10-25 17:38:25 +01:00
1 parent 4d970561fa
commit d3257a1b6b
29 files changed
+202 -35

No files matched your search

+9 -8
View File
@@ -38529,13 +38529,13 @@ components:
account.deletion.not_pending; activity — activity.calendar.future_date, activity.calendar.too_old; ai —
ai.budget.exceeded, ai.provider.none, ai.record.notPermitted, ai.unavailable, ai.usage.aboveCeiling,
ai.usage.defaultAboveCeiling, ai.usage.invalid, ai.usage.levelTooLow, ai.usage.rateLimited; ai_provider
— ai_provider.invalid, ai_provider.no_fields; aiRuns — aiRuns.failed, aiRuns.notFound, aiRuns.timedOut,
aiRuns.workerUnavailable; allergy — allergy.invalid; analytics — analytics.invalid_query; anamnesis —
anamnesis.auditC.invalid, anamnesis.auditC.notFound, anamnesis.auditC.rateLimited,
anamnesis.fact.conflict, anamnesis.fact.currentExists, anamnesis.fact.invalidValue,
anamnesis.lifeConfirm.invalid, anamnesis.lifeSections.invalid; auth — auth.account.deletion_pending,
auth.admin.required, auth.already_authenticated, auth.mfa.code_invalid, auth.missing,
auth.refresh.invalid, auth.refresh.reuse, auth.refresh.revoked, auth.reproof.failed,
— ai_provider.invalid, ai_provider.no_fields; aiRuns — aiRuns.failed, aiRuns.notFound,
aiRuns.readerBusy, aiRuns.timedOut, aiRuns.workerUnavailable; allergy — allergy.invalid; analytics —
analytics.invalid_query; anamnesis — anamnesis.auditC.invalid, anamnesis.auditC.notFound,
anamnesis.auditC.rateLimited, anamnesis.fact.conflict, anamnesis.fact.currentExists,
anamnesis.fact.invalidValue, anamnesis.lifeConfirm.invalid, anamnesis.lifeSections.invalid; auth —
auth.account.deletion_pending, auth.admin.required, auth.already_authenticated, auth.mfa.code_invalid,
auth.missing, auth.refresh.invalid, auth.refresh.reuse, auth.refresh.revoked, auth.reproof.failed,
auth.reproof.required, auth.reproof.too_weak, auth.scope.insufficient, auth.session.insecure_transport,
auth.session.unavailable, auth.stepup.required, auth.token.expired, auth.token.invalid; backup —
backup.foreign_reference, backup.key.missing, backup.payload.undecryptable, backup.restore.active,
@@ -75840,7 +75840,8 @@ components:
description: "Present once `status` is `FAILED`: the `meta.errorCode`, message and HTTP status the queuing route would
have answered with, or one of the run's own codes: `aiRuns.workerUnavailable` (no worker took the run within
fifteen minutes), `aiRuns.timedOut` (the read outlived the calls the AI response time allows),
`aiRuns.failed` (anything else)."
`aiRuns.readerBusy` (the server's document reader was busy past the wait a read gets, or stopped before it
answered; nothing was read or charged, try again in a minute), `aiRuns.failed` (anything else)."
createdAt:
type: string
startedAt:
+5 -1
View File
@@ -175,6 +175,9 @@
"failed": "Das Etikett konnte nicht gelesen werden. Versuch ein schärferes Foto."
}
}
},
"aiRuns": {
"readerBusy": "Der Dokumentleser ist gerade mit anderen Aufgaben beschäftigt. Versuch es in einer Minute noch einmal."
}
},
"doctorReport": {
@@ -13258,7 +13261,8 @@
"backgroundScan": "Wird im Hintergrund gelesen. Mit einem langsamen Modell kann das ein paar Minuten dauern.",
"waitingForWorker": "Warte auf den Hintergrunddienst. Ändert sich das nicht, läuft der Worker auf dem Server vielleicht nicht.",
"errorWorkerUnavailable": "Der Hintergrunddienst ist gerade nicht erreichbar. Versuch es gleich noch einmal oder bitte die Person, die den Server betreibt, ihn zu prüfen.",
"errorTimedOut": "Das Lesen hat länger gedauert, als deine KI-Antwortzeit erlaubt. Versuch es noch einmal oder erhöhe die Antwortzeit in den KI-Einstellungen."
"errorTimedOut": "Das Lesen hat länger gedauert, als deine KI-Antwortzeit erlaubt. Versuch es noch einmal oder erhöhe die Antwortzeit in den KI-Einstellungen.",
"errorReaderBusy": "Der Dokumentleser ist gerade mit anderen Aufgaben beschäftigt. Versuch es in einer Minute noch einmal."
},
"symptoms": {
"add": "Neues Symptom",
+5 -1
View File
@@ -175,6 +175,9 @@
"failed": "The label could not be read. Try a sharper photo."
}
}
},
"aiRuns": {
"readerBusy": "The document reader is busy with other work. Try again in a minute."
}
},
"doctorReport": {
@@ -13258,7 +13261,8 @@
"backgroundScan": "Reading in the background. With a slow model this can take a few minutes.",
"waitingForWorker": "Waiting for the background worker. If this does not change, the server's worker may not be running.",
"errorWorkerUnavailable": "The background worker is not available right now. Try again in a moment, or ask the server operator to check it.",
"errorTimedOut": "Reading took longer than your AI response time allows. Try again, or raise the response time in the AI settings."
"errorTimedOut": "Reading took longer than your AI response time allows. Try again, or raise the response time in the AI settings.",
"errorReaderBusy": "The document reader is busy with other work. Try again in a minute."
},
"symptoms": {
"add": "New symptom",
+5 -1
View File
@@ -175,6 +175,9 @@
"failed": "No se pudo leer la etiqueta. Prueba con una foto más nítida."
}
}
},
"aiRuns": {
"readerBusy": "El lector de documentos está ocupado con otras tareas. Inténtalo de nuevo en un minuto."
}
},
"doctorReport": {
@@ -13258,7 +13261,8 @@
"backgroundScan": "Leyendo en segundo plano. Con un modelo lento puede tardar unos minutos.",
"waitingForWorker": "Esperando al proceso en segundo plano. Si no cambia, puede que el worker del servidor no esté en marcha.",
"errorWorkerUnavailable": "El proceso en segundo plano no está disponible ahora mismo. Inténtalo de nuevo en un momento o pide a quien administra el servidor que lo revise.",
"errorTimedOut": "La lectura tardó más de lo que permite tu tiempo de respuesta de IA. Inténtalo de nuevo o aumenta el tiempo de respuesta en los ajustes de IA."
"errorTimedOut": "La lectura tardó más de lo que permite tu tiempo de respuesta de IA. Inténtalo de nuevo o aumenta el tiempo de respuesta en los ajustes de IA.",
"errorReaderBusy": "El lector de documentos está ocupado con otras tareas. Inténtalo de nuevo en un minuto."
},
"symptoms": {
"add": "Nuevo síntoma",
+5 -1
View File
@@ -175,6 +175,9 @@
"failed": "L’étiquette n’a pas pu être lue. Essayez une photo plus nette."
}
}
},
"aiRuns": {
"readerBusy": "Le lecteur de documents est occupé par d'autres tâches. Réessayez dans une minute."
}
},
"doctorReport": {
@@ -13258,7 +13261,8 @@
"backgroundScan": "Lecture en arrière-plan. Avec un modèle lent, cela peut prendre quelques minutes.",
"waitingForWorker": "En attente du traitement en arrière-plan. Si rien ne change, le worker du serveur n'est peut-être pas lancé.",
"errorWorkerUnavailable": "Le traitement en arrière-plan n'est pas disponible pour le moment. Réessayez dans un instant ou demandez à la personne qui gère le serveur de le vérifier.",
"errorTimedOut": "La lecture a pris plus de temps que votre délai de réponse IA ne le permet. Réessayez ou augmentez ce délai dans les réglages IA."
"errorTimedOut": "La lecture a pris plus de temps que votre délai de réponse IA ne le permet. Réessayez ou augmentez ce délai dans les réglages IA.",
"errorReaderBusy": "Le lecteur de documents est occupé par d'autres tâches. Réessayez dans une minute."
},
"symptoms": {
"add": "Nouveau symptôme",
+5 -1
View File
@@ -175,6 +175,9 @@
"failed": "Non è stato possibile leggere l'etichetta. Prova con una foto più nitida."
}
}
},
"aiRuns": {
"readerBusy": "Il lettore di documenti è occupato con altre attività. Riprova tra un minuto."
}
},
"doctorReport": {
@@ -13258,7 +13261,8 @@
"backgroundScan": "Lettura in background. Con un modello lento può richiedere qualche minuto.",
"waitingForWorker": "In attesa del processo in background. Se non cambia, il worker del server potrebbe non essere attivo.",
"errorWorkerUnavailable": "Il processo in background non è disponibile al momento. Riprova tra poco o chiedi a chi gestisce il server di controllarlo.",
"errorTimedOut": "La lettura ha richiesto più tempo di quanto consenta il tuo tempo di risposta dell'IA. Riprova o aumenta il tempo di risposta nelle impostazioni IA."
"errorTimedOut": "La lettura ha richiesto più tempo di quanto consenta il tuo tempo di risposta dell'IA. Riprova o aumenta il tempo di risposta nelle impostazioni IA.",
"errorReaderBusy": "Il lettore di documenti è occupato con altre attività. Riprova tra un minuto."
},
"symptoms": {
"add": "Nuovo sintomo",
+5 -1
View File
@@ -175,6 +175,9 @@
"failed": "라벨을 읽지 못했어요. 더 선명한 사진으로 다시 시도하세요."
}
}
},
"aiRuns": {
"readerBusy": "문서 판독기가 다른 작업으로 바빠요. 1분 후에 다시 시도하세요."
}
},
"doctorReport": {
@@ -13258,7 +13261,8 @@
"backgroundScan": "백그라운드에서 읽고 있어요. 느린 모델이면 몇 분 걸릴 수 있어요.",
"waitingForWorker": "백그라운드 작업자를 기다리고 있어요. 계속 그대로라면 서버의 작업자가 실행 중이 아닐 수 있어요.",
"errorWorkerUnavailable": "지금은 백그라운드 작업자를 사용할 수 없어요. 잠시 후 다시 시도하거나 서버 운영자에게 확인을 요청하세요.",
"errorTimedOut": "AI 응답 시간 설정보다 읽기가 오래 걸렸어요. 다시 시도하거나 AI 설정에서 응답 시간을 늘려 주세요."
"errorTimedOut": "AI 응답 시간 설정보다 읽기가 오래 걸렸어요. 다시 시도하거나 AI 설정에서 응답 시간을 늘려 주세요.",
"errorReaderBusy": "문서 판독기가 다른 작업으로 바빠요. 1분 후에 다시 시도하세요."
},
"symptoms": {
"add": "새 증상",
+5 -1
View File
@@ -175,6 +175,9 @@
"failed": "Het etiket kon niet worden gelezen. Probeer een scherpere foto."
}
}
},
"aiRuns": {
"readerBusy": "De documentlezer is met ander werk bezig. Probeer het over een minuut opnieuw."
}
},
"doctorReport": {
@@ -13258,7 +13261,8 @@
"backgroundScan": "Wordt op de achtergrond gelezen. Met een traag model kan dit een paar minuten duren.",
"waitingForWorker": "Wachten op het achtergrondproces. Als dit niet verandert, draait het achtergrondproces van de server mogelijk niet.",
"errorWorkerUnavailable": "Het achtergrondproces is op dit moment niet beschikbaar. Probeer het zo opnieuw, of vraag de beheerder van de server om het te controleren.",
"errorTimedOut": "Het lezen duurde langer dan je ingestelde AI-responstijd toestaat. Probeer het opnieuw, of verhoog de responstijd in de AI-instellingen."
"errorTimedOut": "Het lezen duurde langer dan je ingestelde AI-responstijd toestaat. Probeer het opnieuw, of verhoog de responstijd in de AI-instellingen.",
"errorReaderBusy": "De documentlezer is met ander werk bezig. Probeer het over een minuut opnieuw."
},
"symptoms": {
"add": "Nieuw symptoom",
+5 -1
View File
@@ -175,6 +175,9 @@
"failed": "Nie udało się odczytać etykiety. Spróbuj ostrzejszego zdjęcia."
}
}
},
"aiRuns": {
"readerBusy": "Czytnik dokumentów jest zajęty innymi zadaniami. Spróbuj ponownie za minutę."
}
},
"doctorReport": {
@@ -13258,7 +13261,8 @@
"backgroundScan": "Odczyt w tle. Przy wolnym modelu może to potrwać kilka minut.",
"waitingForWorker": "Oczekiwanie na proces w tle. Jeśli nic się nie zmienia, worker na serwerze może nie działać.",
"errorWorkerUnavailable": "Proces w tle jest teraz niedostępny. Spróbuj ponownie za chwilę albo poproś administratora serwera o sprawdzenie go.",
"errorTimedOut": "Odczyt trwał dłużej, niż pozwala ustawiony czas odpowiedzi AI. Spróbuj ponownie albo wydłuż czas odpowiedzi w ustawieniach AI."
"errorTimedOut": "Odczyt trwał dłużej, niż pozwala ustawiony czas odpowiedzi AI. Spróbuj ponownie albo wydłuż czas odpowiedzi w ustawieniach AI.",
"errorReaderBusy": "Czytnik dokumentów jest zajęty innymi zadaniami. Spróbuj ponownie za minutę."
},
"symptoms": {
"add": "Nowy objaw",
+5 -1
View File
@@ -175,6 +175,9 @@
"failed": "Não foi possível ler o rótulo. Tente uma foto mais nítida."
}
}
},
"aiRuns": {
"readerBusy": "O leitor de documentos está ocupado com outras tarefas. Tente novamente em um minuto."
}
},
"doctorReport": {
@@ -13258,7 +13261,8 @@
"backgroundScan": "Lendo em segundo plano. Com um modelo lento, isso pode levar alguns minutos.",
"waitingForWorker": "Aguardando o worker em segundo plano. Se isso não mudar, o worker do servidor pode não estar em execução.",
"errorWorkerUnavailable": "O worker em segundo plano não está disponível agora. Tente novamente em instantes, ou peça ao operador do servidor para verificá-lo.",
"errorTimedOut": "A leitura demorou mais do que o seu tempo de resposta de IA permite. Tente novamente, ou aumente o tempo de resposta nas configurações de IA."
"errorTimedOut": "A leitura demorou mais do que o seu tempo de resposta de IA permite. Tente novamente, ou aumente o tempo de resposta nas configurações de IA.",
"errorReaderBusy": "O leitor de documentos está ocupado com outras tarefas. Tente novamente em um minuto."
},
"symptoms": {
"add": "Novo sintoma",
@@ -228,6 +228,8 @@ export function documentAiErrorKey(err: unknown): string {
return "aiRuns.errorWorkerUnavailable";
case "aiRuns.timedOut":
return "aiRuns.errorTimedOut";
case "aiRuns.readerBusy":
return "aiRuns.errorReaderBusy";
case "consent.ai.required":
return "documents.assist.errorConsent";
case "ai.provider.none":
@@ -117,6 +117,8 @@ export function extractErrorMessage(
return t("aiRuns.errorWorkerUnavailable");
case "aiRuns.timedOut":
return t("aiRuns.errorTimedOut");
case "aiRuns.readerBusy":
return t("aiRuns.errorReaderBusy");
default:
break;
}
+10 -2
View File
@@ -154,9 +154,14 @@ export type VisionInput =
// Refs #776 — `rasterFailed` (this PDF would not render on a raster-
// capable build) is distinct from `pdfNeedsAnthropic` (this build /
// provider cannot take a PDF at all): they need different fixes, so
// they must not share a label.
// they must not share a label. `readerBusy` is neither: the reader
// could not take the document now, and the work is tried again.
reason:
"decryptFailed" | "fileType" | "pdfNeedsAnthropic" | "rasterFailed";
| "decryptFailed"
| "fileType"
| "pdfNeedsAnthropic"
| "rasterFailed"
| "readerBusy";
};
/**
@@ -211,6 +216,9 @@ export async function prepareVisionInput(
if (raster.ok) {
return { ok: true, images: raster.images, documents: [] };
}
if (raster.reason === "busy" || raster.reason === "interrupted") {
return { ok: false, reason: "readerBusy" };
}
// Refs #776 — a build that cannot rasterize at all is a capability gap a
// PDF-native (Anthropic) provider would cover; a render failure on a
// capable build is a property of THIS document. Report them apart.
+4 -1
View File
@@ -52,6 +52,7 @@ import { dateOnlyAtNoonUtc } from "@/lib/tz/date-only";
import { refusalAsFailure } from "./http";
import { meteredReadBudget } from "./metered-budget";
import {
readerBusyFailure,
runFailure,
unreadableDocument,
type DocumentTextPick,
@@ -306,7 +307,9 @@ export async function executeDocumentExtract(args: {
source = { mode: "text", text: input.text };
}
const plan = await planInboundReads(source, schedule);
if (!plan.ok) return refused(pdfUnreadable());
if (!plan.ok) {
return refused(plan.busy ? readerBusyFailure() : pdfUnreadable());
}
await schedule.arm(plan.args.chunks.length * OCR_EXTRACT_MODEL_CALLS);
let result: InboundExtractionResult;
+3
View File
@@ -32,6 +32,7 @@ import { recordIndexAttempt } from "@/lib/documents/index-document";
import type { requireDocumentVisionProvider } from "@/lib/documents/provider-order";
import { annotate } from "@/lib/logging/context";
import { readerBusyFailure } from "./run-support";
import { settleAiRunBudget } from "./store";
import type {
AiRunBudget,
@@ -157,6 +158,8 @@ export async function executeDocumentIndex(args: {
// gone.
await settleAiRunBudget(userId, budget, 0, null);
await refundDocumentAiSlot(userId);
// A busy reader says nothing about the document: no attempt is recorded.
if (vision.reason === "readerBusy") return readerBusyFailure();
if (vision.reason === "pdfNeedsAnthropic") {
await recordIndexAttempt(userId, document.id, {
indexed: false,
+15 -1
View File
@@ -13,7 +13,7 @@ import type {
} from "@/lib/documents/provider-order";
import type { VisionInput } from "@/lib/documents/ai-route-support";
import type { AiRunOutcome } from "./types";
import { AI_RUN_ERROR_CODES, type AiRunOutcome } from "./types";
export type DocumentVisionPick = Awaited<
ReturnType<typeof requireDocumentVisionProvider>
@@ -44,6 +44,19 @@ export function unreadableDocument(parseError: boolean): AiRunOutcome<never> {
});
}
/**
* The document reader was busy with other work for longer than a person's
* read waits, or its process ended for no reason it measured. Nothing was
* read and nothing is charged; asking again in a minute is the answer.
*/
export function readerBusyFailure(): AiRunOutcome<never> {
return runFailure({
status: 503,
message: "The document reader is busy. Try again in a minute.",
errorCode: AI_RUN_ERROR_CODES.readerBusy,
});
}
/**
* A stored original that could not become a provider input, as the summary
* and suggest routes have always answered it.
@@ -59,6 +72,7 @@ export function visionPreparationFailure(
errorCode: "documents.inbound.pdfNeedsAnthropic",
});
}
if (reason === "readerBusy") return readerBusyFailure();
if (reason === "fileType") {
return runFailure({
status: 422,
+1
View File
@@ -74,6 +74,7 @@ export const AI_RUN_ERROR_CODES = {
workerUnavailable: "aiRuns.workerUnavailable",
timedOut: "aiRuns.timedOut",
failed: "aiRuns.failed",
readerBusy: "aiRuns.readerBusy",
} as const;
/** How long a queued run waits for a worker before it is failed. */
+2 -2
View File
@@ -484,7 +484,7 @@ export function planInboundTextReads(
export async function planInboundReads(
source: InboundReadSource,
schedule: OcrSchedule,
): Promise<({ ok: true } & InboundReads) | { ok: false }> {
): Promise<({ ok: true } & InboundReads) | { ok: false; busy?: true }> {
if (source.mode === "text") {
return { ok: true, ...planInboundTextReads(source.text, schedule) };
}
@@ -494,7 +494,7 @@ export async function planInboundReads(
source.pdfSupported,
schedule,
);
if (!plan.ok) return { ok: false };
if (!plan.ok) return plan;
return {
ok: true,
args: { mode: "vision", chunks: plan.chunks, totalPages: plan.totalPages },
+7
View File
@@ -58,6 +58,7 @@ import {
import { transcribeDocument } from "@/lib/documents/describe";
import { documentAutoReadEnabled } from "@/lib/documents/document-settings";
import { localExtractText } from "@/lib/documents/local-extract";
import { PdfReaderUnavailableError } from "@/lib/documents/pdfjs";
import { resolveDocumentVisionProvider } from "@/lib/documents/provider-order";
import { decryptDocumentContent } from "@/lib/documents/store";
import { prisma } from "@/lib/db";
@@ -238,6 +239,12 @@ async function tryProviderIndex(
if (vision.reason === "rasterFailed") {
return { outcome: null, note: "raster-failed" };
}
if (vision.reason === "readerBusy") {
// The reader could not take the document now. Falling back to the
// local text layer would settle for the lesser read for good; the job
// runs again instead.
throw new PdfReaderUnavailableError("busy");
}
if (vision.reason === "pdfNeedsAnthropic") {
return { outcome: null, note: "pdf-needs-anthropic" };
}
+12 -2
View File
@@ -170,7 +170,12 @@ export type OcrVisionPlan =
/** Set when the PDF is longer than the page cap: only the first pages are read. */
pageCoverage?: { read: number; total: number };
}
| { ok: false };
/**
* `busy`: the reader could not take the PDF now (busy past the wait a read
* gets, or its process ended for no measured reason). Not a verdict on the
* file; the person asks again.
*/
| { ok: false; busy?: true };
/**
* Plan the reads of an image or a PDF. Every PDF is rendered first: a report
@@ -226,7 +231,12 @@ export async function planVisionReads(
: {}),
};
}
if (!pdfSupported) return { ok: false };
if (!pdfSupported) {
return !raster.ok &&
(raster.reason === "busy" || raster.reason === "interrupted")
? { ok: false, busy: true }
: { ok: false };
}
return {
ok: true,
chunks: [
@@ -285,6 +285,32 @@ describe("runDocumentAiRun", () => {
);
});
it("reads as interactive work of the run's owner, so the document reader serves it first", async () => {
let seen: unknown = null;
executeDocumentIndex.mockImplementation(async () => {
const { currentPdfWork } = await import("@/lib/documents/pdf-isolation");
seen = currentPdfWork();
return { ok: true, data: {} };
});
await runDocumentAiRun("run1", NOW + 2_700_000, now);
expect(seen).toEqual({ owner: "u1", lane: "interactive" });
});
it("answers a busy document reader with a retryable code and hands the reservation back", async () => {
const { PdfReaderUnavailableError } = await import("@/lib/documents/pdfjs");
executeDocumentIndex.mockRejectedValue(
new PdfReaderUnavailableError("busy"),
);
expect(await runDocumentAiRun("run1", NOW + 2_700_000, now)).toBe("failed");
expect(settleAiRunBudget).toHaveBeenCalledWith("u1", budget, 0, null);
expect(failAiRun).toHaveBeenCalledWith(
"run1",
expect.objectContaining({ status: 503, errorCode: "aiRuns.readerBusy" }),
"RUNNING",
new Date(NOW),
);
});
it("fails a run whose document is gone and hands the reservation back", async () => {
loadOwnedDocument.mockResolvedValue(null);
expect(await runDocumentAiRun("run1", NOW + 2_700_000, now)).toBe("failed");
+15 -1
View File
@@ -42,6 +42,8 @@ import { loadOwnedDocument } from "@/lib/documents/ai-route-support";
import { loadDocumentChatText } from "@/lib/documents/content-index";
import { refusalAsFailure } from "@/lib/documents/ai-runs/http";
import { executeDocumentExtract } from "@/lib/documents/ai-runs/extract-run";
import { withPdfWork } from "@/lib/documents/pdf-isolation";
import { PdfReaderUnavailableError } from "@/lib/documents/pdfjs";
import {
DOCUMENT_INDEX_MODEL_CALLS,
executeDocumentIndex,
@@ -614,7 +616,12 @@ export async function runDocumentAiRun(
let outcome: AiRunOutcome<AiRunResult>;
try {
outcome = await RUN_DISPATCH[run.kind](run, deadlineAt, now);
// A person asked for this read and is waiting on it: the document
// reader serves it ahead of background work, in turn with other people.
outcome = await withPdfWork(
{ owner: run.userId, lane: "interactive" },
() => RUN_DISPATCH[run.kind](run, deadlineAt, now),
);
} catch (err) {
// Everything that throws here threw before the body took over the
// reservation, so it goes back here.
@@ -622,6 +629,13 @@ export async function runDocumentAiRun(
const failure: AiRunFailure = (err instanceof RunRefused
? err.failure
: null) ??
(err instanceof PdfReaderUnavailableError
? {
status: 503,
errorCode: AI_RUN_ERROR_CODES.readerBusy,
message: "The document reader is busy. Try again in a minute.",
}
: null) ??
refusalAsFailure(err) ?? {
status: 500,
errorCode: AI_RUN_ERROR_CODES.failed,
@@ -55,6 +55,7 @@ import {
upsertContentIndex,
} from "@/lib/documents/content-index";
import { transcribeDocument } from "@/lib/documents/describe";
import { withPdfWork } from "@/lib/documents/pdf-isolation";
import { getGlobalBoss } from "@/lib/jobs/boss-instance";
import { resolveDocumentVisionProvider } from "@/lib/documents/provider-order";
import { annotate } from "@/lib/logging/context";
@@ -320,7 +321,10 @@ export async function runContentIndexBackfillForUser(
skipped += 1;
continue;
}
const vision = await prepareVisionInput(document, pick.pdfSupported);
const vision = await withPdfWork(
{ owner: userId, lane: "background" },
() => prepareVisionInput(document, pick.pdfSupported),
);
if (!vision.ok) {
// Not vision-indexable after all (undecryptable, unreadable bytes,
// or a PDF the rasteriser could not render) — refund and move on so
+7 -6
View File
@@ -15,6 +15,7 @@
import { STATUS_MODEL_CALLS } from "@/lib/ai/ai-budgets";
import { maybeAutoStageLabFacts } from "@/lib/documents/auto-stage-labs";
import { indexDocumentContent } from "@/lib/documents/index-document";
import { withPdfWork } from "@/lib/documents/pdf-isolation";
import type { OcrSchedule } from "@/lib/documents/ocr-plan";
import { ocrReadCallMs } from "@/lib/documents/ocr-read";
import { prisma } from "@/lib/db";
@@ -50,9 +51,9 @@ export async function runDocumentIndex(
const { userId, documentId } = payload;
if (!userId || !documentId) return;
const localOnly = payload.localOnly === true;
const outcome = await indexDocumentContent(userId, documentId, {
localOnly,
});
const outcome = await withPdfWork({ owner: userId, lane: "background" }, () =>
indexDocumentContent(userId, documentId, { localOnly }),
);
annotate({
action: { name: "documents.autoIndex.run" },
meta: outcome.indexed
@@ -72,9 +73,9 @@ export async function runDocumentIndex(
options.deadlineAt === undefined
? undefined
: await stagingSchedule(userId, options.deadlineAt, options.now);
await maybeAutoStageLabFacts(userId, documentId, { schedule }).catch(
caughtAs("documents.auto_index.lab_staging_failed"),
);
await withPdfWork({ owner: userId, lane: "background" }, () =>
maybeAutoStageLabFacts(userId, documentId, { schedule }),
).catch(caughtAs("documents.auto_index.lab_staging_failed"));
}
}
+22 -1
View File
@@ -43,6 +43,8 @@ import type { Locale } from "@/lib/i18n/config";
import { resolveJobLocale } from "@/lib/i18n/job-locale";
import { documentAutoReadEnabled } from "@/lib/documents/document-settings";
import { resolveDocumentVisionProvider } from "@/lib/documents/provider-order";
import { withPdfWork } from "@/lib/documents/pdf-isolation";
import { PdfReaderUnavailableError } from "@/lib/documents/pdfjs";
import { encryptDocumentSummary } from "@/lib/documents/store";
import { prisma } from "@/lib/db";
import { getGlobalBoss } from "@/lib/jobs/boss-instance";
@@ -81,6 +83,25 @@ export interface DocumentSummaryPayload {
enqueuedAt?: string;
}
/**
* The stored original as provider input, read as this account's background
* work. A busy reader is not a property of the document: the job throws, and
* pg-boss runs it again.
*/
async function prepareSummaryVision(
userId: string,
document: Parameters<typeof prepareVisionInput>[0],
pick: { pdfSupported: boolean },
): Promise<ReturnType<typeof prepareVisionInput>> {
const vision = await withPdfWork({ owner: userId, lane: "background" }, () =>
prepareVisionInput(document, pick.pdfSupported),
);
if (!vision.ok && vision.reason === "readerBusy") {
throw new PdfReaderUnavailableError("busy");
}
return vision;
}
/**
* Generate + persist ONE document's background summary (owner-scoped). Never
* throws for an expected outcome — every gate that fails resolves to a tagged
@@ -210,7 +231,7 @@ export async function runDocumentSummaryJob(
const document = await loadOwnedDocument(userId, documentId);
if (!document) return;
const vision = await prepareVisionInput(document, pick.pdfSupported);
const vision = await prepareSummaryVision(userId, document, pick);
if (!vision.ok) {
// Cannot read this file with the picked provider (decrypt miss, unsupported
// type, or a PDF needing an Anthropic provider). Graceful no-op.
+5
View File
@@ -20,6 +20,7 @@ import { AI_BUDGETS } from "@/lib/ai/ai-budgets";
import { refusalAsFailure } from "@/lib/documents/ai-runs/http";
import type { ProviderChainType } from "@/lib/ai/provider-chain";
import { meteredReadBudget } from "@/lib/documents/ai-runs/metered-budget";
import { readerBusyFailure } from "@/lib/documents/ai-runs/run-support";
import {
AI_RUN_ERROR_CODES,
type AiRunBudget,
@@ -221,6 +222,10 @@ export async function executeOcrExtraction(args: {
input.pick.pdfSupported,
schedule,
);
if (!plan.ok && plan.busy) {
await settle(false);
return readerBusyFailure();
}
if (!plan.ok) {
annotate({
action: { name: "labs.ocr.fileRejected" },
@@ -29,6 +29,7 @@ import {
type AIProvider,
type CompletionParams,
} from "@/lib/ai/types";
import { readerBusyFailure } from "@/lib/documents/ai-runs/run-support";
import { settleAiRunBudget } from "@/lib/documents/ai-runs/store";
import type {
AiRunBudget,
@@ -232,6 +233,13 @@ export async function executeSupplementLabelExtraction(args: {
];
} else {
const raster = await rasterizePdf(bytes);
if (
!raster.ok &&
(raster.reason === "busy" || raster.reason === "interrupted")
) {
await settleAiRunBudget(userId, budget, 0, null);
return readerBusyFailure();
}
if (!raster.ok) {
await settleAiRunBudget(userId, budget, 0, null);
return failure({
+1
View File
@@ -57,6 +57,7 @@ export const ERROR_CODE_CATALOGUE: Readonly<Record<string, readonly string[]>> =
aiRuns: [
"aiRuns.failed",
"aiRuns.notFound",
"aiRuns.readerBusy",
"aiRuns.timedOut",
"aiRuns.workerUnavailable",
],
+1 -1
View File
@@ -64,7 +64,7 @@ const aiRun = z
})
.nullable()
.describe(
"Present once `status` is `FAILED`: the `meta.errorCode`, message and HTTP status the queuing route would have answered with, or one of the run's own codes: `aiRuns.workerUnavailable` (no worker took the run within fifteen minutes), `aiRuns.timedOut` (the read outlived the calls the AI response time allows), `aiRuns.failed` (anything else).",
"Present once `status` is `FAILED`: the `meta.errorCode`, message and HTTP status the queuing route would have answered with, or one of the run's own codes: `aiRuns.workerUnavailable` (no worker took the run within fifteen minutes), `aiRuns.timedOut` (the read outlived the calls the AI response time allows), `aiRuns.readerBusy` (the server's document reader was busy past the wait a read gets, or stopped before it answered; nothing was read or charged, try again in a minute), `aiRuns.failed` (anything else).",
),
createdAt: z.string(),
startedAt: z.string().nullable(),