fix(documents): keep document reads inside their job's expiry

Since the lab staging reads a long report in parts, one auto-index could
make the transcription plus up to twenty reads of two calls each at the
person's response timeout, hours past pg-boss's fifteen-minute default,
which failed the job and retried it beside the reads still running. The
staging now takes the run's deadline as a schedule and plans only the
reads that fit before it. The index job is long, with an expiry above the
calls that run before the plan can say no, a stop on jobDeadline and a
lock per document; the document run hands the same staging its own
deadline, which moves the run's expiry as the reads are planned. The
document summary, one call and its reasoning repeat, gets an expiry above
that worst case.

ocrReadCallMs in the read core is the one per-call timeout both deadlines
divide by. The expiry guard now names every importer of the document and
lab read entry points with the queues that reach it.

The runtime guard's planted cases stripped every file's comments again on
each rescan, which took over a second alone and timed out under a full
parallel run; the stripped text is now memoised.

Refs #1274
This commit is contained in:
Marc-André Bombeck committed 2026-10-24 11:28:57 +02:00
1 parent 96eeec6080
commit a1ee2ac003
21 files changed
+434 -76

No files matched your search

@@ -173,6 +173,27 @@ describe("maybeAutoStageLabFacts", () => {
expect(created.data.every((f) => f.status === "PENDING")).toBe(true);
});
it("plans a long report's reads against the run's deadline (#1274)", async () => {
// Four hundred lines of a panel, many reads of text.
const longReport = Array.from(
{ length: 400 },
(_, i) => `Hämoglobin ${i} g/dl Referenzbereich 13-17 LDL 120 mg/dl`,
).join("\n");
mockText.mockResolvedValue({ text: longReport, source: "verbatim" });
await maybeAutoStageLabFacts("u1", "d1");
const unbounded = mockExtract.mock.calls[0]![0].chunks.length;
expect(unbounded).toBeGreaterThan(1);
const arm = vi.fn(async () => {});
await maybeAutoStageLabFacts("u1", "d1", {
schedule: { maxModelCalls: () => 2, arm },
});
// Two calls left before the deadline: one read (a call and its retry).
expect(mockExtract.mock.calls[1]![0].chunks).toHaveLength(1);
expect(arm).toHaveBeenCalledWith(2);
});
it("skips (manual path only) when a module is off", async () => {
mockModule.mockImplementation(async (_u, key) => key !== "labs");
const res = await maybeAutoStageLabFacts("u1", "d1");
+9 -3
View File
@@ -12,6 +12,7 @@
* (status, message, `documents.inbound.*` code); the caller renders it as a
* response or stores it on the run.
*/
import type { OcrSchedule } from "@/lib/documents/ocr-plan";
import { AI_BUDGETS } from "@/lib/ai/ai-budgets";
import type { ProviderChainType } from "@/lib/ai/provider-chain";
import { auditLog } from "@/lib/auth/audit";
@@ -56,6 +57,11 @@ export const DOCUMENT_INDEX_RESERVE_TOKENS =
export interface IndexOrigin {
ipAddress: string | null;
worker: boolean;
/**
* A background run's deadline, which the lab staging after the index plans
* its reads against. A request has none: it reads as before.
*/
schedule?: OcrSchedule;
}
export type DocumentIndexInput =
@@ -109,9 +115,9 @@ async function finishIndex(
// modules on, still STORED with no facts, provider + consent, looks like a
// lab report) — a non-lab document is a tagged no-op, and a staging failure
// never fails the index that just succeeded.
const staging = await maybeAutoStageLabFacts(userId, documentId).catch(
() => null,
);
const staging = await maybeAutoStageLabFacts(userId, documentId, {
schedule: origin.schedule,
}).catch(() => null);
const labFactsStaged = staging?.staged === true ? staging.facts : 0;
return {
ok: true,
+16 -1
View File
@@ -41,6 +41,10 @@ import {
runInboundExtraction,
type InboundExtractionResult,
} from "@/lib/documents/extract";
import {
OCR_EXTRACT_MODEL_CALLS,
type OcrSchedule,
} from "@/lib/documents/ocr-plan";
import { OcrExtractError } from "@/lib/documents/ocr-read";
import { encryptFactData, encryptFactProvenance } from "@/lib/documents/store";
import { prisma } from "@/lib/db";
@@ -251,10 +255,16 @@ async function stageFactsIfStored(
* Auto-stage lab facts for one freshly indexed document (owner-scoped). Called
* fire-and-forget by the auto-index job after a successful index; every guard
* miss returns a tagged no-op so the manual extract button stays the fallback.
*
* `schedule` is the run's deadline when a job reads (the auto-index job, the
* document run): the report's text is then split into as many reads as fit
* before it, instead of up to `OCR_MAX_PAGES` reads of two calls each that a
* slow model could stretch past the job's expiry (#1274).
*/
export async function maybeAutoStageLabFacts(
userId: string,
documentId: string,
options: { schedule?: OcrSchedule } = {},
): Promise<AutoStageOutcome> {
const [inboundOn, labsOn] = await Promise.all([
isModuleEnabled(userId, "inboundDocuments"),
@@ -321,9 +331,14 @@ export async function maybeAutoStageLabFacts(
{ cap, surface: "job" },
);
const plan = planInboundTextReads(chat.text, undefined);
const plan = planInboundTextReads(chat.text, options.schedule);
let result: InboundExtractionResult;
try {
// A run that cannot fit even one read before its deadline refuses here;
// the reservation is settled below like any other failed read.
await options.schedule?.arm(
plan.args.chunks.length * OCR_EXTRACT_MODEL_CALLS,
);
result = await runInboundExtraction({
userId,
provider: provider.pick.entry.instance,
+15
View File
@@ -25,6 +25,7 @@
* The raw image / PDF bytes live in memory only; nothing here logs them.
*/
import { AI_BUDGETS } from "@/lib/ai/ai-budgets";
import { callTimeoutMs } from "@/lib/ai/effective-timeout";
import { aiEgressRefusal } from "@/lib/ai/capabilities/egress";
import type { AiUnavailableError } from "@/lib/ai/capabilities/refusal";
import {
@@ -69,6 +70,20 @@ export function ocrReadParams(prompt: {
};
}
/**
* The timeout one call of a read runs under for a record whose setting is
* `responseTimeoutSeconds`: the read's own default, lifted by the setting.
* What a deadline that plans reads divides its time by.
*/
export function ocrReadCallMs(
responseTimeoutSeconds: number | null | undefined,
): number {
return callTimeoutMs(
{ timeoutMs: AI_BUDGETS.ocrExtract.timeoutMs },
responseTimeoutSeconds,
);
}
/** Appended to the user turn when the first answer was not the schema. */
const RETRY_SUFFIX =
"\n\nYour previous response was not valid JSON matching the schema. Return ONLY the JSON object described in the system prompt, with no prose or markdown.";