mirror of
https://github.com/MBombeck/HealthLog.git
synced 2026-11-18 08:02:12 +00:00
fix(documents): tell people when the document reader is busy, and attribute its work
A read a person started now runs as interactive work of their account, and the document jobs as background work of theirs, so the reader's queue can order and cap them. When the reader is busy or stopped before it answered, a lab scan, label read or document read fails with aiRuns.readerBusy (503, nothing read or charged) instead of a misleading unreadable-file error, and the index and summary jobs throw so the queue runs them again rather than recording an outcome about the document.
This commit is contained in:
1 parent
4d970561fa
commit
d3257a1b6b
29 files changed
+202
-35
No files matched your search
@@ -228,6 +228,8 @@ export function documentAiErrorKey(err: unknown): string {
|
||||
return "aiRuns.errorWorkerUnavailable";
|
||||
case "aiRuns.timedOut":
|
||||
return "aiRuns.errorTimedOut";
|
||||
case "aiRuns.readerBusy":
|
||||
return "aiRuns.errorReaderBusy";
|
||||
case "consent.ai.required":
|
||||
return "documents.assist.errorConsent";
|
||||
case "ai.provider.none":
|
||||
|
||||
@@ -117,6 +117,8 @@ export function extractErrorMessage(
|
||||
return t("aiRuns.errorWorkerUnavailable");
|
||||
case "aiRuns.timedOut":
|
||||
return t("aiRuns.errorTimedOut");
|
||||
case "aiRuns.readerBusy":
|
||||
return t("aiRuns.errorReaderBusy");
|
||||
default:
|
||||
break;
|
||||
}
|
||||
|
||||
@@ -154,9 +154,14 @@ export type VisionInput =
|
||||
// Refs #776 — `rasterFailed` (this PDF would not render on a raster-
|
||||
// capable build) is distinct from `pdfNeedsAnthropic` (this build /
|
||||
// provider cannot take a PDF at all): they need different fixes, so
|
||||
// they must not share a label.
|
||||
// they must not share a label. `readerBusy` is neither: the reader
|
||||
// could not take the document now, and the work is tried again.
|
||||
reason:
|
||||
"decryptFailed" | "fileType" | "pdfNeedsAnthropic" | "rasterFailed";
|
||||
| "decryptFailed"
|
||||
| "fileType"
|
||||
| "pdfNeedsAnthropic"
|
||||
| "rasterFailed"
|
||||
| "readerBusy";
|
||||
};
|
||||
|
||||
/**
|
||||
@@ -211,6 +216,9 @@ export async function prepareVisionInput(
|
||||
if (raster.ok) {
|
||||
return { ok: true, images: raster.images, documents: [] };
|
||||
}
|
||||
if (raster.reason === "busy" || raster.reason === "interrupted") {
|
||||
return { ok: false, reason: "readerBusy" };
|
||||
}
|
||||
// Refs #776 — a build that cannot rasterize at all is a capability gap a
|
||||
// PDF-native (Anthropic) provider would cover; a render failure on a
|
||||
// capable build is a property of THIS document. Report them apart.
|
||||
|
||||
@@ -52,6 +52,7 @@ import { dateOnlyAtNoonUtc } from "@/lib/tz/date-only";
|
||||
import { refusalAsFailure } from "./http";
|
||||
import { meteredReadBudget } from "./metered-budget";
|
||||
import {
|
||||
readerBusyFailure,
|
||||
runFailure,
|
||||
unreadableDocument,
|
||||
type DocumentTextPick,
|
||||
@@ -306,7 +307,9 @@ export async function executeDocumentExtract(args: {
|
||||
source = { mode: "text", text: input.text };
|
||||
}
|
||||
const plan = await planInboundReads(source, schedule);
|
||||
if (!plan.ok) return refused(pdfUnreadable());
|
||||
if (!plan.ok) {
|
||||
return refused(plan.busy ? readerBusyFailure() : pdfUnreadable());
|
||||
}
|
||||
await schedule.arm(plan.args.chunks.length * OCR_EXTRACT_MODEL_CALLS);
|
||||
|
||||
let result: InboundExtractionResult;
|
||||
|
||||
@@ -32,6 +32,7 @@ import { recordIndexAttempt } from "@/lib/documents/index-document";
|
||||
import type { requireDocumentVisionProvider } from "@/lib/documents/provider-order";
|
||||
import { annotate } from "@/lib/logging/context";
|
||||
|
||||
import { readerBusyFailure } from "./run-support";
|
||||
import { settleAiRunBudget } from "./store";
|
||||
import type {
|
||||
AiRunBudget,
|
||||
@@ -157,6 +158,8 @@ export async function executeDocumentIndex(args: {
|
||||
// gone.
|
||||
await settleAiRunBudget(userId, budget, 0, null);
|
||||
await refundDocumentAiSlot(userId);
|
||||
// A busy reader says nothing about the document: no attempt is recorded.
|
||||
if (vision.reason === "readerBusy") return readerBusyFailure();
|
||||
if (vision.reason === "pdfNeedsAnthropic") {
|
||||
await recordIndexAttempt(userId, document.id, {
|
||||
indexed: false,
|
||||
|
||||
@@ -13,7 +13,7 @@ import type {
|
||||
} from "@/lib/documents/provider-order";
|
||||
import type { VisionInput } from "@/lib/documents/ai-route-support";
|
||||
|
||||
import type { AiRunOutcome } from "./types";
|
||||
import { AI_RUN_ERROR_CODES, type AiRunOutcome } from "./types";
|
||||
|
||||
export type DocumentVisionPick = Awaited<
|
||||
ReturnType<typeof requireDocumentVisionProvider>
|
||||
@@ -44,6 +44,19 @@ export function unreadableDocument(parseError: boolean): AiRunOutcome<never> {
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* The document reader was busy with other work for longer than a person's
|
||||
* read waits, or its process ended for no reason it measured. Nothing was
|
||||
* read and nothing is charged; asking again in a minute is the answer.
|
||||
*/
|
||||
export function readerBusyFailure(): AiRunOutcome<never> {
|
||||
return runFailure({
|
||||
status: 503,
|
||||
message: "The document reader is busy. Try again in a minute.",
|
||||
errorCode: AI_RUN_ERROR_CODES.readerBusy,
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* A stored original that could not become a provider input, as the summary
|
||||
* and suggest routes have always answered it.
|
||||
@@ -59,6 +72,7 @@ export function visionPreparationFailure(
|
||||
errorCode: "documents.inbound.pdfNeedsAnthropic",
|
||||
});
|
||||
}
|
||||
if (reason === "readerBusy") return readerBusyFailure();
|
||||
if (reason === "fileType") {
|
||||
return runFailure({
|
||||
status: 422,
|
||||
|
||||
@@ -74,6 +74,7 @@ export const AI_RUN_ERROR_CODES = {
|
||||
workerUnavailable: "aiRuns.workerUnavailable",
|
||||
timedOut: "aiRuns.timedOut",
|
||||
failed: "aiRuns.failed",
|
||||
readerBusy: "aiRuns.readerBusy",
|
||||
} as const;
|
||||
|
||||
/** How long a queued run waits for a worker before it is failed. */
|
||||
|
||||
@@ -484,7 +484,7 @@ export function planInboundTextReads(
|
||||
export async function planInboundReads(
|
||||
source: InboundReadSource,
|
||||
schedule: OcrSchedule,
|
||||
): Promise<({ ok: true } & InboundReads) | { ok: false }> {
|
||||
): Promise<({ ok: true } & InboundReads) | { ok: false; busy?: true }> {
|
||||
if (source.mode === "text") {
|
||||
return { ok: true, ...planInboundTextReads(source.text, schedule) };
|
||||
}
|
||||
@@ -494,7 +494,7 @@ export async function planInboundReads(
|
||||
source.pdfSupported,
|
||||
schedule,
|
||||
);
|
||||
if (!plan.ok) return { ok: false };
|
||||
if (!plan.ok) return plan;
|
||||
return {
|
||||
ok: true,
|
||||
args: { mode: "vision", chunks: plan.chunks, totalPages: plan.totalPages },
|
||||
|
||||
@@ -58,6 +58,7 @@ import {
|
||||
import { transcribeDocument } from "@/lib/documents/describe";
|
||||
import { documentAutoReadEnabled } from "@/lib/documents/document-settings";
|
||||
import { localExtractText } from "@/lib/documents/local-extract";
|
||||
import { PdfReaderUnavailableError } from "@/lib/documents/pdfjs";
|
||||
import { resolveDocumentVisionProvider } from "@/lib/documents/provider-order";
|
||||
import { decryptDocumentContent } from "@/lib/documents/store";
|
||||
import { prisma } from "@/lib/db";
|
||||
@@ -238,6 +239,12 @@ async function tryProviderIndex(
|
||||
if (vision.reason === "rasterFailed") {
|
||||
return { outcome: null, note: "raster-failed" };
|
||||
}
|
||||
if (vision.reason === "readerBusy") {
|
||||
// The reader could not take the document now. Falling back to the
|
||||
// local text layer would settle for the lesser read for good; the job
|
||||
// runs again instead.
|
||||
throw new PdfReaderUnavailableError("busy");
|
||||
}
|
||||
if (vision.reason === "pdfNeedsAnthropic") {
|
||||
return { outcome: null, note: "pdf-needs-anthropic" };
|
||||
}
|
||||
|
||||
@@ -170,7 +170,12 @@ export type OcrVisionPlan =
|
||||
/** Set when the PDF is longer than the page cap: only the first pages are read. */
|
||||
pageCoverage?: { read: number; total: number };
|
||||
}
|
||||
| { ok: false };
|
||||
/**
|
||||
* `busy`: the reader could not take the PDF now (busy past the wait a read
|
||||
* gets, or its process ended for no measured reason). Not a verdict on the
|
||||
* file; the person asks again.
|
||||
*/
|
||||
| { ok: false; busy?: true };
|
||||
|
||||
/**
|
||||
* Plan the reads of an image or a PDF. Every PDF is rendered first: a report
|
||||
@@ -226,7 +231,12 @@ export async function planVisionReads(
|
||||
: {}),
|
||||
};
|
||||
}
|
||||
if (!pdfSupported) return { ok: false };
|
||||
if (!pdfSupported) {
|
||||
return !raster.ok &&
|
||||
(raster.reason === "busy" || raster.reason === "interrupted")
|
||||
? { ok: false, busy: true }
|
||||
: { ok: false };
|
||||
}
|
||||
return {
|
||||
ok: true,
|
||||
chunks: [
|
||||
|
||||
@@ -285,6 +285,32 @@ describe("runDocumentAiRun", () => {
|
||||
);
|
||||
});
|
||||
|
||||
it("reads as interactive work of the run's owner, so the document reader serves it first", async () => {
|
||||
let seen: unknown = null;
|
||||
executeDocumentIndex.mockImplementation(async () => {
|
||||
const { currentPdfWork } = await import("@/lib/documents/pdf-isolation");
|
||||
seen = currentPdfWork();
|
||||
return { ok: true, data: {} };
|
||||
});
|
||||
await runDocumentAiRun("run1", NOW + 2_700_000, now);
|
||||
expect(seen).toEqual({ owner: "u1", lane: "interactive" });
|
||||
});
|
||||
|
||||
it("answers a busy document reader with a retryable code and hands the reservation back", async () => {
|
||||
const { PdfReaderUnavailableError } = await import("@/lib/documents/pdfjs");
|
||||
executeDocumentIndex.mockRejectedValue(
|
||||
new PdfReaderUnavailableError("busy"),
|
||||
);
|
||||
expect(await runDocumentAiRun("run1", NOW + 2_700_000, now)).toBe("failed");
|
||||
expect(settleAiRunBudget).toHaveBeenCalledWith("u1", budget, 0, null);
|
||||
expect(failAiRun).toHaveBeenCalledWith(
|
||||
"run1",
|
||||
expect.objectContaining({ status: 503, errorCode: "aiRuns.readerBusy" }),
|
||||
"RUNNING",
|
||||
new Date(NOW),
|
||||
);
|
||||
});
|
||||
|
||||
it("fails a run whose document is gone and hands the reservation back", async () => {
|
||||
loadOwnedDocument.mockResolvedValue(null);
|
||||
expect(await runDocumentAiRun("run1", NOW + 2_700_000, now)).toBe("failed");
|
||||
|
||||
@@ -42,6 +42,8 @@ import { loadOwnedDocument } from "@/lib/documents/ai-route-support";
|
||||
import { loadDocumentChatText } from "@/lib/documents/content-index";
|
||||
import { refusalAsFailure } from "@/lib/documents/ai-runs/http";
|
||||
import { executeDocumentExtract } from "@/lib/documents/ai-runs/extract-run";
|
||||
import { withPdfWork } from "@/lib/documents/pdf-isolation";
|
||||
import { PdfReaderUnavailableError } from "@/lib/documents/pdfjs";
|
||||
import {
|
||||
DOCUMENT_INDEX_MODEL_CALLS,
|
||||
executeDocumentIndex,
|
||||
@@ -614,7 +616,12 @@ export async function runDocumentAiRun(
|
||||
|
||||
let outcome: AiRunOutcome<AiRunResult>;
|
||||
try {
|
||||
outcome = await RUN_DISPATCH[run.kind](run, deadlineAt, now);
|
||||
// A person asked for this read and is waiting on it: the document
|
||||
// reader serves it ahead of background work, in turn with other people.
|
||||
outcome = await withPdfWork(
|
||||
{ owner: run.userId, lane: "interactive" },
|
||||
() => RUN_DISPATCH[run.kind](run, deadlineAt, now),
|
||||
);
|
||||
} catch (err) {
|
||||
// Everything that throws here threw before the body took over the
|
||||
// reservation, so it goes back here.
|
||||
@@ -622,6 +629,13 @@ export async function runDocumentAiRun(
|
||||
const failure: AiRunFailure = (err instanceof RunRefused
|
||||
? err.failure
|
||||
: null) ??
|
||||
(err instanceof PdfReaderUnavailableError
|
||||
? {
|
||||
status: 503,
|
||||
errorCode: AI_RUN_ERROR_CODES.readerBusy,
|
||||
message: "The document reader is busy. Try again in a minute.",
|
||||
}
|
||||
: null) ??
|
||||
refusalAsFailure(err) ?? {
|
||||
status: 500,
|
||||
errorCode: AI_RUN_ERROR_CODES.failed,
|
||||
|
||||
@@ -55,6 +55,7 @@ import {
|
||||
upsertContentIndex,
|
||||
} from "@/lib/documents/content-index";
|
||||
import { transcribeDocument } from "@/lib/documents/describe";
|
||||
import { withPdfWork } from "@/lib/documents/pdf-isolation";
|
||||
import { getGlobalBoss } from "@/lib/jobs/boss-instance";
|
||||
import { resolveDocumentVisionProvider } from "@/lib/documents/provider-order";
|
||||
import { annotate } from "@/lib/logging/context";
|
||||
@@ -320,7 +321,10 @@ export async function runContentIndexBackfillForUser(
|
||||
skipped += 1;
|
||||
continue;
|
||||
}
|
||||
const vision = await prepareVisionInput(document, pick.pdfSupported);
|
||||
const vision = await withPdfWork(
|
||||
{ owner: userId, lane: "background" },
|
||||
() => prepareVisionInput(document, pick.pdfSupported),
|
||||
);
|
||||
if (!vision.ok) {
|
||||
// Not vision-indexable after all (undecryptable, unreadable bytes,
|
||||
// or a PDF the rasteriser could not render) — refund and move on so
|
||||
|
||||
@@ -15,6 +15,7 @@
|
||||
import { STATUS_MODEL_CALLS } from "@/lib/ai/ai-budgets";
|
||||
import { maybeAutoStageLabFacts } from "@/lib/documents/auto-stage-labs";
|
||||
import { indexDocumentContent } from "@/lib/documents/index-document";
|
||||
import { withPdfWork } from "@/lib/documents/pdf-isolation";
|
||||
import type { OcrSchedule } from "@/lib/documents/ocr-plan";
|
||||
import { ocrReadCallMs } from "@/lib/documents/ocr-read";
|
||||
import { prisma } from "@/lib/db";
|
||||
@@ -50,9 +51,9 @@ export async function runDocumentIndex(
|
||||
const { userId, documentId } = payload;
|
||||
if (!userId || !documentId) return;
|
||||
const localOnly = payload.localOnly === true;
|
||||
const outcome = await indexDocumentContent(userId, documentId, {
|
||||
localOnly,
|
||||
});
|
||||
const outcome = await withPdfWork({ owner: userId, lane: "background" }, () =>
|
||||
indexDocumentContent(userId, documentId, { localOnly }),
|
||||
);
|
||||
annotate({
|
||||
action: { name: "documents.autoIndex.run" },
|
||||
meta: outcome.indexed
|
||||
@@ -72,9 +73,9 @@ export async function runDocumentIndex(
|
||||
options.deadlineAt === undefined
|
||||
? undefined
|
||||
: await stagingSchedule(userId, options.deadlineAt, options.now);
|
||||
await maybeAutoStageLabFacts(userId, documentId, { schedule }).catch(
|
||||
caughtAs("documents.auto_index.lab_staging_failed"),
|
||||
);
|
||||
await withPdfWork({ owner: userId, lane: "background" }, () =>
|
||||
maybeAutoStageLabFacts(userId, documentId, { schedule }),
|
||||
).catch(caughtAs("documents.auto_index.lab_staging_failed"));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -43,6 +43,8 @@ import type { Locale } from "@/lib/i18n/config";
|
||||
import { resolveJobLocale } from "@/lib/i18n/job-locale";
|
||||
import { documentAutoReadEnabled } from "@/lib/documents/document-settings";
|
||||
import { resolveDocumentVisionProvider } from "@/lib/documents/provider-order";
|
||||
import { withPdfWork } from "@/lib/documents/pdf-isolation";
|
||||
import { PdfReaderUnavailableError } from "@/lib/documents/pdfjs";
|
||||
import { encryptDocumentSummary } from "@/lib/documents/store";
|
||||
import { prisma } from "@/lib/db";
|
||||
import { getGlobalBoss } from "@/lib/jobs/boss-instance";
|
||||
@@ -81,6 +83,25 @@ export interface DocumentSummaryPayload {
|
||||
enqueuedAt?: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* The stored original as provider input, read as this account's background
|
||||
* work. A busy reader is not a property of the document: the job throws, and
|
||||
* pg-boss runs it again.
|
||||
*/
|
||||
async function prepareSummaryVision(
|
||||
userId: string,
|
||||
document: Parameters<typeof prepareVisionInput>[0],
|
||||
pick: { pdfSupported: boolean },
|
||||
): Promise<ReturnType<typeof prepareVisionInput>> {
|
||||
const vision = await withPdfWork({ owner: userId, lane: "background" }, () =>
|
||||
prepareVisionInput(document, pick.pdfSupported),
|
||||
);
|
||||
if (!vision.ok && vision.reason === "readerBusy") {
|
||||
throw new PdfReaderUnavailableError("busy");
|
||||
}
|
||||
return vision;
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate + persist ONE document's background summary (owner-scoped). Never
|
||||
* throws for an expected outcome — every gate that fails resolves to a tagged
|
||||
@@ -210,7 +231,7 @@ export async function runDocumentSummaryJob(
|
||||
const document = await loadOwnedDocument(userId, documentId);
|
||||
if (!document) return;
|
||||
|
||||
const vision = await prepareVisionInput(document, pick.pdfSupported);
|
||||
const vision = await prepareSummaryVision(userId, document, pick);
|
||||
if (!vision.ok) {
|
||||
// Cannot read this file with the picked provider (decrypt miss, unsupported
|
||||
// type, or a PDF needing an Anthropic provider). Graceful no-op.
|
||||
|
||||
@@ -20,6 +20,7 @@ import { AI_BUDGETS } from "@/lib/ai/ai-budgets";
|
||||
import { refusalAsFailure } from "@/lib/documents/ai-runs/http";
|
||||
import type { ProviderChainType } from "@/lib/ai/provider-chain";
|
||||
import { meteredReadBudget } from "@/lib/documents/ai-runs/metered-budget";
|
||||
import { readerBusyFailure } from "@/lib/documents/ai-runs/run-support";
|
||||
import {
|
||||
AI_RUN_ERROR_CODES,
|
||||
type AiRunBudget,
|
||||
@@ -221,6 +222,10 @@ export async function executeOcrExtraction(args: {
|
||||
input.pick.pdfSupported,
|
||||
schedule,
|
||||
);
|
||||
if (!plan.ok && plan.busy) {
|
||||
await settle(false);
|
||||
return readerBusyFailure();
|
||||
}
|
||||
if (!plan.ok) {
|
||||
annotate({
|
||||
action: { name: "labs.ocr.fileRejected" },
|
||||
|
||||
@@ -29,6 +29,7 @@ import {
|
||||
type AIProvider,
|
||||
type CompletionParams,
|
||||
} from "@/lib/ai/types";
|
||||
import { readerBusyFailure } from "@/lib/documents/ai-runs/run-support";
|
||||
import { settleAiRunBudget } from "@/lib/documents/ai-runs/store";
|
||||
import type {
|
||||
AiRunBudget,
|
||||
@@ -232,6 +233,13 @@ export async function executeSupplementLabelExtraction(args: {
|
||||
];
|
||||
} else {
|
||||
const raster = await rasterizePdf(bytes);
|
||||
if (
|
||||
!raster.ok &&
|
||||
(raster.reason === "busy" || raster.reason === "interrupted")
|
||||
) {
|
||||
await settleAiRunBudget(userId, budget, 0, null);
|
||||
return readerBusyFailure();
|
||||
}
|
||||
if (!raster.ok) {
|
||||
await settleAiRunBudget(userId, budget, 0, null);
|
||||
return failure({
|
||||
|
||||
@@ -57,6 +57,7 @@ export const ERROR_CODE_CATALOGUE: Readonly<Record<string, readonly string[]>> =
|
||||
aiRuns: [
|
||||
"aiRuns.failed",
|
||||
"aiRuns.notFound",
|
||||
"aiRuns.readerBusy",
|
||||
"aiRuns.timedOut",
|
||||
"aiRuns.workerUnavailable",
|
||||
],
|
||||
|
||||
@@ -64,7 +64,7 @@ const aiRun = z
|
||||
})
|
||||
.nullable()
|
||||
.describe(
|
||||
"Present once `status` is `FAILED`: the `meta.errorCode`, message and HTTP status the queuing route would have answered with, or one of the run's own codes: `aiRuns.workerUnavailable` (no worker took the run within fifteen minutes), `aiRuns.timedOut` (the read outlived the calls the AI response time allows), `aiRuns.failed` (anything else).",
|
||||
"Present once `status` is `FAILED`: the `meta.errorCode`, message and HTTP status the queuing route would have answered with, or one of the run's own codes: `aiRuns.workerUnavailable` (no worker took the run within fifteen minutes), `aiRuns.timedOut` (the read outlived the calls the AI response time allows), `aiRuns.readerBusy` (the server's document reader was busy past the wait a read gets, or stopped before it answered; nothing was read or charged, try again in a minute), `aiRuns.failed` (anything else).",
|
||||
),
|
||||
createdAt: z.string(),
|
||||
startedAt: z.string().nullable(),
|
||||
|
||||
Reference in new issue
Block a user