mirror of
https://github.com/MBombeck/HealthLog.git
synced 2026-11-14 06:40:50 +00:00
fix(labs): name a failed re-check as an error and close the last budget gaps
A re-check or reservation that failed itself stopped the scan as "unavailable", and the review screen said AI had been turned off. It now stops as stoppedBy error with its own message in every locale, and the failure is reported as one: an error signal and the worker error reporter. A scan that failed that way before any row answers 500 aiRuns.failed. The overspend booked before a further read is now net of the cached share, as the settlement books it. A reservation that names no known owner stops the scan instead of taking the user plan's larger cap. A page's text layer is cut at the last whole line; only a page whose first line alone runs past the cap keeps part of it, cut at a space or hard, rather than collapsing to the bare cut mark, and an endless line is no longer read past the cap. The unused per-call text limit is gone. The run result's description names every outcome a stopped scan can have.
This commit is contained in:
1 parent
e897549dff
commit
4c50f48b26
19 files changed
+279
-75
No files matched your search
@@ -306,6 +306,9 @@ describe("<OcrReadNotes> — what a read could not give", () => {
|
||||
expect(render({ unreadPages: [4], stoppedBy: "unavailable" })).toContain(
|
||||
"AI was turned off while the report was being read",
|
||||
);
|
||||
const failed = render({ unreadPages: [4], stoppedBy: "error" });
|
||||
expect(failed).toContain("because of a server error");
|
||||
expect(failed).not.toContain("AI was turned off");
|
||||
});
|
||||
|
||||
it("says how many readings the report had when it ran past the cap", () => {
|
||||
|
||||
@@ -548,7 +548,9 @@ export function OcrReadNotes({ notes }: { notes: OcrReadNotesValue }) {
|
||||
<p>
|
||||
{stoppedBy === "budget"
|
||||
? t("labs.ocr.stoppedBudget")
|
||||
: t("labs.ocr.stoppedUnavailable")}
|
||||
: stoppedBy === "error"
|
||||
? t("labs.ocr.stoppedError")
|
||||
: t("labs.ocr.stoppedUnavailable")}
|
||||
</p>
|
||||
) : null}
|
||||
{unreadPages ? (
|
||||
|
||||
@@ -5,8 +5,10 @@ import { describe, expect, it, vi } from "vitest";
|
||||
vi.mock("@/lib/logging/context", () => ({ annotate: vi.fn() }));
|
||||
|
||||
import {
|
||||
pageText,
|
||||
RASTER_HARD_MAX_PAGES,
|
||||
RASTER_MAX_PAGES,
|
||||
RASTER_PAGE_TEXT_MAX,
|
||||
rasterizePdf,
|
||||
TEXT_LAYER_CUT_MARK,
|
||||
} from "../rasterize-pdf";
|
||||
@@ -21,17 +23,29 @@ import {
|
||||
* so the test carries no binary blobs.
|
||||
*/
|
||||
function buildPdf(pageCount: number, withText: boolean): Buffer {
|
||||
return buildPdfFromContents(
|
||||
Array.from({ length: pageCount }, (_, p) =>
|
||||
withText
|
||||
? `BT /F1 24 Tf 40 120 Td (Page ${p + 1} lab report glucose creatinine) Tj ET`
|
||||
: `0 0 1 rg 40 40 120 120 re f`,
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
/** A PDF with one page per content stream, on the given media box. */
|
||||
function buildPdfFromContents(
|
||||
contents: string[],
|
||||
mediaBox = "0 0 200 200",
|
||||
): Buffer {
|
||||
const pageCount = contents.length;
|
||||
const pageNums: number[] = [];
|
||||
const perPage: { pageNum: number; contentNum: number; content: string }[] =
|
||||
[];
|
||||
let n = 4;
|
||||
for (let p = 0; p < pageCount; p++) {
|
||||
for (const content of contents) {
|
||||
const pageNum = n++;
|
||||
const contentNum = n++;
|
||||
pageNums.push(pageNum);
|
||||
const content = withText
|
||||
? `BT /F1 24 Tf 40 120 Td (Page ${p + 1} lab report glucose creatinine) Tj ET`
|
||||
: `0 0 1 rg 40 40 120 120 re f`;
|
||||
perPage.push({ pageNum, contentNum, content });
|
||||
}
|
||||
|
||||
@@ -43,7 +57,7 @@ function buildPdf(pageCount: number, withText: boolean): Buffer {
|
||||
bodies[3] = `<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>`;
|
||||
for (const { pageNum, contentNum, content } of perPage) {
|
||||
bodies[pageNum] =
|
||||
`<< /Type /Page /Parent 2 0 R /MediaBox [0 0 200 200] /Resources << /Font << /F1 3 0 R >> >> /Contents ${contentNum} 0 R >>`;
|
||||
`<< /Type /Page /Parent 2 0 R /MediaBox [${mediaBox}] /Resources << /Font << /F1 3 0 R >> >> /Contents ${contentNum} 0 R >>`;
|
||||
bodies[contentNum] =
|
||||
`<< /Length ${Buffer.byteLength(content, "latin1")} >>\nstream\n${content}\nendstream`;
|
||||
}
|
||||
@@ -111,17 +125,61 @@ describe("rasterizePdf", () => {
|
||||
expect(plain.ok && plain.texts).toBeUndefined();
|
||||
});
|
||||
|
||||
it("stops collecting a page's text at the cap while reading, and marks the cut", async () => {
|
||||
const capped = await rasterizePdf(buildPdf(1, true), 20, {
|
||||
withText: true,
|
||||
textMaxChars: 8,
|
||||
it("stops collecting a page's text at the cap while reading, cut at a line and marked", async () => {
|
||||
// 900 short lines on one tall page: past RASTER_PAGE_TEXT_MAX.
|
||||
const lines = Array.from(
|
||||
{ length: 900 },
|
||||
(_, i) => `(Marker ${String(i).padStart(4, "0")} value 12,34) Tj T*`,
|
||||
);
|
||||
const pdf = buildPdfFromContents(
|
||||
// The filled box gives the page ink, so it is not taken for a blank.
|
||||
[
|
||||
`0 0 1 rg 0 0 200 3000 re f 0 g BT /F1 3 Tf 3 TL 5 2995 Td ${lines.join(" ")} ET`,
|
||||
],
|
||||
"0 0 200 3000",
|
||||
);
|
||||
const result = await rasterizePdf(pdf, 1, { withText: true });
|
||||
expect(result.ok).toBe(true);
|
||||
if (!result.ok) return;
|
||||
const text = result.texts?.[0] ?? "";
|
||||
expect(text.length).toBeLessThanOrEqual(RASTER_PAGE_TEXT_MAX);
|
||||
const kept = text.split("\n");
|
||||
expect(kept.at(-1)).toBe(TEXT_LAYER_CUT_MARK);
|
||||
expect(kept[0]).toBe("Marker 0000 value 12,34");
|
||||
expect(kept.at(-2)).toMatch(/^Marker \d{4} value 12,34$/);
|
||||
});
|
||||
|
||||
it("hard-cuts a page that is one endless line instead of keeping only the mark", async () => {
|
||||
const endless = "1".repeat(16_000);
|
||||
const pdf = buildPdfFromContents(
|
||||
[
|
||||
`0 0 1 rg 0 0 20000 200 re f 0 g BT /F1 1 Tf 0 100 Td (${endless}) Tj ET`,
|
||||
],
|
||||
"0 0 20000 200",
|
||||
);
|
||||
const result = await rasterizePdf(pdf, 1, { withText: true });
|
||||
const text = (result.ok && result.texts?.[0]) || "";
|
||||
expect(text.length).toBeLessThanOrEqual(RASTER_PAGE_TEXT_MAX);
|
||||
expect(text.endsWith(TEXT_LAYER_CUT_MARK)).toBe(true);
|
||||
expect(text.length).toBeGreaterThan(RASTER_PAGE_TEXT_MAX / 2);
|
||||
});
|
||||
|
||||
it("stops reading a line that never ends once it passes the cap", async () => {
|
||||
// One text item after another with no line end: past the cap the rest
|
||||
// is not read into memory at all.
|
||||
let read = 0;
|
||||
const items = Array.from({ length: 5000 }, () => ({
|
||||
get str() {
|
||||
read += 1;
|
||||
return "123456789";
|
||||
},
|
||||
hasEOL: false,
|
||||
}));
|
||||
const text = await pageText({
|
||||
getTextContent: async () => ({ items }),
|
||||
});
|
||||
expect(capped.ok && capped.texts).toEqual([TEXT_LAYER_CUT_MARK]);
|
||||
const roomy = await rasterizePdf(buildPdf(1, true), 20, {
|
||||
withText: true,
|
||||
textMaxChars: 1000,
|
||||
});
|
||||
expect(roomy.ok && roomy.texts?.[0]).toMatch(/^Page 1 lab/);
|
||||
expect(text.endsWith(TEXT_LAYER_CUT_MARK)).toBe(true);
|
||||
expect(read).toBeLessThan(RASTER_PAGE_TEXT_MAX / 5);
|
||||
});
|
||||
|
||||
it("lets a caller ask for more pages, never past RASTER_HARD_MAX_PAGES", async () => {
|
||||
|
||||
@@ -295,35 +295,65 @@ export const RASTER_PAGE_TEXT_MAX = 12_000;
|
||||
export const TEXT_LAYER_CUT_MARK = "[text layer cut here]";
|
||||
|
||||
/**
|
||||
* A page's text layer, line by line: what the PDF states as characters.
|
||||
* Empty for a page without one or on any failure; cut at a line, and marked,
|
||||
* past `maxChars`.
|
||||
* The head of `text` that fits `room` characters: cut at the last space before
|
||||
* the limit so no number is cut in half, or hard at the limit when the text
|
||||
* has no space to cut at.
|
||||
*/
|
||||
async function pageText(page: PdfPage, maxChars: number): Promise<string> {
|
||||
function headThatFits(text: string, room: number): string {
|
||||
if (text.length <= room) return text;
|
||||
const head = text.slice(0, room);
|
||||
const space = head.lastIndexOf(" ");
|
||||
return space > 0 ? head.slice(0, space) : head;
|
||||
}
|
||||
|
||||
/**
|
||||
* A page's text layer, line by line: what the PDF states as characters.
|
||||
* Empty for a page without one or on any failure. Past
|
||||
* `RASTER_PAGE_TEXT_MAX` it is cut (at a line, else a space, else hard) and
|
||||
* marked.
|
||||
*/
|
||||
export async function pageText(
|
||||
page: Pick<PdfPage, "getTextContent">,
|
||||
): Promise<string> {
|
||||
const max = RASTER_PAGE_TEXT_MAX;
|
||||
try {
|
||||
const { items } = await page.getTextContent();
|
||||
const lines: string[] = [];
|
||||
let length = 0;
|
||||
let line = "";
|
||||
// Cut at the last whole line; only a page whose first line alone runs
|
||||
// past the cap keeps part of that line, cut at a space or hard.
|
||||
const cut = (): string => {
|
||||
if (lines.length === 0) {
|
||||
const room = max - TEXT_LAYER_CUT_MARK.length - 1;
|
||||
const head = headThatFits(line.replace(/\s+/g, " ").trim(), room);
|
||||
if (head.trim()) lines.push(head.trim());
|
||||
}
|
||||
lines.push(TEXT_LAYER_CUT_MARK);
|
||||
return lines.join("\n");
|
||||
};
|
||||
/** Adds the line in hand; false when it does not fit. */
|
||||
const flush = (): boolean => {
|
||||
const clean = line.replace(/\s+/g, " ").trim();
|
||||
line = "";
|
||||
if (!clean) return true;
|
||||
if (length + clean.length + 1 > maxChars) return false;
|
||||
if (!clean) {
|
||||
line = "";
|
||||
return true;
|
||||
}
|
||||
if (length + clean.length + 1 > max - TEXT_LAYER_CUT_MARK.length - 1) {
|
||||
return false;
|
||||
}
|
||||
lines.push(clean);
|
||||
length += clean.length + 1;
|
||||
line = "";
|
||||
return true;
|
||||
};
|
||||
for (const item of items) {
|
||||
line += `${item.str ?? ""} `;
|
||||
// A line that runs on without a break is flushed at the cap, so one
|
||||
// A line that runs on without a break is judged at the cap, so one
|
||||
// endless line cannot grow past it either.
|
||||
if ((item.hasEOL || line.length > maxChars) && !flush()) {
|
||||
lines.push(TEXT_LAYER_CUT_MARK);
|
||||
return lines.join("\n");
|
||||
}
|
||||
if ((item.hasEOL || line.length > max) && !flush()) return cut();
|
||||
}
|
||||
if (!flush()) lines.push(TEXT_LAYER_CUT_MARK);
|
||||
if (!flush()) return cut();
|
||||
return lines.join("\n");
|
||||
} catch {
|
||||
return "";
|
||||
@@ -340,7 +370,7 @@ async function pageText(page: PdfPage, maxChars: number): Promise<string> {
|
||||
export async function rasterizePdf(
|
||||
buffer: Buffer,
|
||||
maxPages: number = RASTER_MAX_PAGES,
|
||||
opts: { withText?: boolean; textMaxChars?: number } = {},
|
||||
opts: { withText?: boolean } = {},
|
||||
): Promise<RasterResult> {
|
||||
let doc: PdfDocument | null = null;
|
||||
let task: PdfLoadingTask | null = null;
|
||||
@@ -428,11 +458,7 @@ export async function rasterizePdf(
|
||||
dataBase64: jpeg.toString("base64"),
|
||||
});
|
||||
factory.destroy?.(cc);
|
||||
if (opts.withText) {
|
||||
texts.push(
|
||||
await pageText(page, opts.textMaxChars ?? RASTER_PAGE_TEXT_MAX),
|
||||
);
|
||||
}
|
||||
if (opts.withText) texts.push(await pageText(page));
|
||||
} finally {
|
||||
page.cleanup();
|
||||
}
|
||||
|
||||
@@ -10,6 +10,10 @@
|
||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
|
||||
vi.mock("@/lib/logging/context", () => ({ annotate: vi.fn() }));
|
||||
vi.mock("@/lib/logging/signal", () => ({ emitSignal: vi.fn() }));
|
||||
vi.mock("@/lib/jobs/report-worker-error", () => ({
|
||||
reportWorkerError: vi.fn().mockResolvedValue(undefined),
|
||||
}));
|
||||
vi.mock("@/lib/ai/capabilities/egress", () => ({
|
||||
aiEgressRefusal: vi.fn().mockResolvedValue(null),
|
||||
}));
|
||||
@@ -25,6 +29,8 @@ vi.mock("@/lib/db", () => ({
|
||||
}));
|
||||
|
||||
import { aiEgressRefusal } from "@/lib/ai/capabilities/egress";
|
||||
import { reportWorkerError } from "@/lib/jobs/report-worker-error";
|
||||
import { emitSignal } from "@/lib/logging/signal";
|
||||
import { AiUnavailableError } from "@/lib/ai/capabilities/refusal";
|
||||
import type { AIProvider, CompletionResult } from "@/lib/ai/types";
|
||||
import {
|
||||
@@ -324,7 +330,7 @@ describe("asked again before every read", () => {
|
||||
expect((err as OcrExtractError).reason).toBe("budget");
|
||||
});
|
||||
|
||||
it("treats a re-check that fails as a stop, keeping the pages read", async () => {
|
||||
it("treats a re-check that fails as a stop of its own, kept and reported", async () => {
|
||||
vi.mocked(aiEgressRefusal)
|
||||
.mockResolvedValueOnce(null)
|
||||
.mockRejectedValueOnce(new Error("connection reset"));
|
||||
@@ -332,7 +338,19 @@ describe("asked again before every read", () => {
|
||||
const out = await run(p, [page(1), page(2), page(3)], 3);
|
||||
expect(out.data.rows.map((r) => r.analyte)).toEqual(["Hb"]);
|
||||
expect(out.data.unreadPages).toEqual([2, 3]);
|
||||
expect(out.data.stoppedBy).toBe("unavailable");
|
||||
expect(out.data.stoppedBy).toBe("error");
|
||||
expect(emitSignal).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
action: "labs.ocr.recheckFailed",
|
||||
level: "error",
|
||||
meta: { phase: "egress", readsDone: 1, reads: 3 },
|
||||
}),
|
||||
);
|
||||
expect(reportWorkerError).toHaveBeenCalledWith(
|
||||
"labs-ocr-recheck",
|
||||
expect.any(Error),
|
||||
{ phase: "egress", readsDone: 1, reads: 3 },
|
||||
);
|
||||
|
||||
const reserveRead = vi.fn().mockRejectedValue(new Error("deadlock"));
|
||||
const second = provider(reply(envelope([row("Hb", 14)])));
|
||||
@@ -340,7 +358,19 @@ describe("asked again before every read", () => {
|
||||
reserveRead,
|
||||
});
|
||||
expect(out2.data.unreadPages).toEqual([2]);
|
||||
expect(out2.data.stoppedBy).toBe("unavailable");
|
||||
expect(out2.data.stoppedBy).toBe("error");
|
||||
expect(reportWorkerError).toHaveBeenLastCalledWith(
|
||||
"labs-ocr-recheck",
|
||||
expect.any(Error),
|
||||
{ phase: "budget", readsDone: 1, reads: 2 },
|
||||
);
|
||||
});
|
||||
|
||||
it("fails as an error, not an unreadable page, when the first check fails", async () => {
|
||||
vi.mocked(aiEgressRefusal).mockRejectedValueOnce(new Error("timeout"));
|
||||
const { provider: p } = provider();
|
||||
const err = await run(p, [page(1)], 1).catch((e: unknown) => e);
|
||||
expect((err as OcrExtractError).reason).toBe("error");
|
||||
});
|
||||
|
||||
it("counts what the calls cost even when the scan then fails", async () => {
|
||||
@@ -531,6 +561,10 @@ describe("what one read sends", () => {
|
||||
const cut = cutAtLine("Hb 14,2 g/dl 13,5 - 17,5 MCV 90,1 fl", 30);
|
||||
expect(cut.length).toBeLessThanOrEqual(30);
|
||||
expect(cut).toBe(`Hb 14,2\n${TEXT_LAYER_CUT_MARK}`);
|
||||
// No line break and no space to cut at: cut hard, never down to the mark.
|
||||
expect(cutAtLine("1".repeat(100), 40)).toBe(
|
||||
`${"1".repeat(18)}\n${TEXT_LAYER_CUT_MARK}`,
|
||||
);
|
||||
expect(cutAtLine("Hb 14,2\nMCV 90,1 fl 80 - 100 fl and more", 31)).toBe(
|
||||
`Hb 14,2\n${TEXT_LAYER_CUT_MARK}`,
|
||||
);
|
||||
|
||||
@@ -310,6 +310,37 @@ describe("text mode", () => {
|
||||
);
|
||||
});
|
||||
|
||||
it("judges the overspend net of the cached share, as the settlement books it", async () => {
|
||||
vi.mocked(reserveBudget).mockResolvedValue({
|
||||
allowed: true,
|
||||
reserved: 1,
|
||||
} as never);
|
||||
vi.mocked(runOcrExtraction).mockImplementation(async (args) => {
|
||||
args.meter!.spent += 20_000;
|
||||
args.meter!.cached += 4000;
|
||||
await args.reserveRead!();
|
||||
return { data: { rows: [] } as never, spentTokens: 0, cachedTokens: 0 };
|
||||
});
|
||||
await executeOcrExtraction({ userId: "u1", input: text(), budget });
|
||||
// 20 000 spent, 4 000 of it cached: 16 000 net against 1 500 reserved.
|
||||
expect(vi.mocked(reserveBudget).mock.calls[0][1]).toBe(14_500);
|
||||
});
|
||||
|
||||
it("stops rather than takes the larger cap when the reservation names no known owner", async () => {
|
||||
let answer: boolean | undefined;
|
||||
vi.mocked(runOcrExtraction).mockImplementation(async (args) => {
|
||||
answer = await args.reserveRead!();
|
||||
return { data: { rows: [] } as never, spentTokens: 0, cachedTokens: 0 };
|
||||
});
|
||||
await executeOcrExtraction({
|
||||
userId: "u1",
|
||||
input: text(),
|
||||
budget: { ...budget, owner: undefined as never },
|
||||
});
|
||||
expect(answer).toBe(false);
|
||||
expect(reserveBudget).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("answers a scan stopped before any row with the stop's own reason", async () => {
|
||||
vi.mocked(runOcrExtraction).mockRejectedValueOnce(
|
||||
new OcrExtractError("x", "budget"),
|
||||
@@ -330,7 +361,7 @@ describe("text mode", () => {
|
||||
).toMatchObject({ ok: false, errorCode: "consent.ai.required" });
|
||||
|
||||
vi.mocked(runOcrExtraction).mockRejectedValueOnce(
|
||||
new OcrExtractError("x", "unavailable"),
|
||||
new OcrExtractError("x", "error"),
|
||||
);
|
||||
expect(
|
||||
await executeOcrExtraction({ userId: "u1", input: text(), budget }),
|
||||
|
||||
+34
-14
@@ -53,7 +53,9 @@ import {
|
||||
} from "@/lib/labs/biomarker-resolve";
|
||||
import { sameLabUnit } from "@/lib/labs/unit-normalise";
|
||||
import { TEXT_LAYER_CUT_MARK } from "@/lib/documents/rasterize-pdf";
|
||||
import { reportWorkerError } from "@/lib/jobs/report-worker-error";
|
||||
import { annotate } from "@/lib/logging/context";
|
||||
import { emitSignal } from "@/lib/logging/signal";
|
||||
import {
|
||||
parseExtractedLabs,
|
||||
type ExtractedLabs,
|
||||
@@ -132,7 +134,7 @@ const RETRY_SUFFIX =
|
||||
|
||||
/** Why a read produced nothing usable. */
|
||||
export type OcrExtractFailureReason =
|
||||
"unparseable" | "output_limit" | "budget" | "unavailable";
|
||||
"unparseable" | "output_limit" | "budget" | "unavailable" | "error";
|
||||
|
||||
export class OcrExtractError extends Error {
|
||||
constructor(
|
||||
@@ -141,8 +143,9 @@ export class OcrExtractError extends Error {
|
||||
* `output_limit`: every read stopped at the provider's output limit
|
||||
* before it returned a single complete row. `unparseable`: the provider
|
||||
* answered, but not with the schema, even after the corrective retry.
|
||||
* `budget` / `unavailable`: the scan stopped (the day's budget, AI turned
|
||||
* off for the record) before any read produced a row.
|
||||
* `budget` / `unavailable` / `error`: the scan stopped (the day's
|
||||
* budget, AI turned off for the record, a check that failed) before any
|
||||
* read produced a row.
|
||||
*/
|
||||
readonly reason: OcrExtractFailureReason = "unparseable",
|
||||
/** With `unavailable`: the wire's own refusal, when it gave one. */
|
||||
@@ -444,9 +447,11 @@ export interface OcrChunk {
|
||||
/**
|
||||
* Why a scan stopped before its last read. `budget`: the day's AI budget ran
|
||||
* out. `unavailable`: AI was turned off for the record (the operator's
|
||||
* switch, the module, a withdrawn consent) while the scan ran.
|
||||
* switch, the module, a withdrawn consent) while the scan ran. `error`: the
|
||||
* check before a read failed itself (the database, most likely), so the scan
|
||||
* stopped closed rather than read on unchecked.
|
||||
*/
|
||||
export type OcrStopReason = "budget" | "unavailable";
|
||||
export type OcrStopReason = "budget" | "unavailable" | "error";
|
||||
|
||||
export interface RunOcrExtractionArgs {
|
||||
userId: string;
|
||||
@@ -482,15 +487,19 @@ export interface OcrExtraction {
|
||||
export const OCR_READ_TEXT_MAX = 12_000;
|
||||
|
||||
/**
|
||||
* `text` cut to at most `max` characters at the last line break (else the
|
||||
* last space) before the limit, so no number is cut in half, and marked.
|
||||
* `text` cut to at most `max` characters and marked: at the last line break
|
||||
* before the limit, else the last space (so no number is cut in half), else
|
||||
* hard at the limit when the text has neither.
|
||||
*/
|
||||
export function cutAtLine(text: string, max: number): string {
|
||||
if (text.length <= max) return text;
|
||||
const room = Math.max(0, max - TEXT_LAYER_CUT_MARK.length - 1);
|
||||
const head = text.slice(0, room);
|
||||
const at = Math.max(head.lastIndexOf("\n"), head.lastIndexOf(" "));
|
||||
return `${at > 0 ? head.slice(0, at) : ""}\n${TEXT_LAYER_CUT_MARK}`.trimStart();
|
||||
const line = head.lastIndexOf("\n");
|
||||
const space = head.lastIndexOf(" ");
|
||||
const kept =
|
||||
line > 0 ? head.slice(0, line) : space > 0 ? head.slice(0, space) : head;
|
||||
return `${kept}\n${TEXT_LAYER_CUT_MARK}`.trimStart();
|
||||
}
|
||||
|
||||
/** The fenced text-layer hint for a read's pages, or "" when none has one. */
|
||||
@@ -613,8 +622,10 @@ export async function runOcrExtraction(
|
||||
// PDF takes a while): AI turned off for the record, or a consent
|
||||
// withdrawn, stops the scan here; so does the day's budget for every
|
||||
// read after the first, which the route already reserved. A check that
|
||||
// itself fails stops the scan the same way, closed, with what was read.
|
||||
// itself fails stops the scan too, closed, with what was read, and is
|
||||
// reported as the failure it is.
|
||||
let stop: OcrStopReason | null = null;
|
||||
let phase: "egress" | "budget" = "egress";
|
||||
try {
|
||||
const refused = await aiEgressRefusal("labsOcr", args.userId, [
|
||||
args.providerType,
|
||||
@@ -622,11 +633,20 @@ export async function runOcrExtraction(
|
||||
if (refused) {
|
||||
refusal = refused;
|
||||
stop = "unavailable";
|
||||
} else if (i > 0 && args.reserveRead && !(await args.reserveRead())) {
|
||||
stop = "budget";
|
||||
} else if (i > 0 && args.reserveRead) {
|
||||
phase = "budget";
|
||||
if (!(await args.reserveRead())) stop = "budget";
|
||||
}
|
||||
} catch {
|
||||
stop = "unavailable";
|
||||
} catch (err) {
|
||||
stop = "error";
|
||||
const meta = { phase, readsDone: i, reads: chunks.length };
|
||||
emitSignal({
|
||||
action: "labs.ocr.recheckFailed",
|
||||
level: "error",
|
||||
error: err,
|
||||
meta,
|
||||
});
|
||||
void reportWorkerError("labs-ocr-recheck", err, meta);
|
||||
}
|
||||
if (stop) {
|
||||
stoppedBy = stop;
|
||||
|
||||
+26
-12
@@ -203,18 +203,23 @@ function unreadable(err: OcrExtractError): AiRunOutcome<never> {
|
||||
errorCode: "labs.ocr.budgetExceeded",
|
||||
});
|
||||
case "unavailable": {
|
||||
// The wire's own refusal when it gave one (the client reads its code:
|
||||
// consent, the operator's switch, the module); else the re-check
|
||||
// itself failed, which is the background read failing.
|
||||
// The wire's own refusal (the client reads its code: consent, the
|
||||
// operator's switch, the module).
|
||||
const refused = err.refusal ? refusalAsFailure(err.refusal) : null;
|
||||
return refused
|
||||
? failure(refused)
|
||||
: failure({
|
||||
status: 500,
|
||||
message: "The background read failed.",
|
||||
errorCode: AI_RUN_ERROR_CODES.failed,
|
||||
});
|
||||
if (refused) return failure(refused);
|
||||
return failure({
|
||||
status: 500,
|
||||
message: "The background read failed.",
|
||||
errorCode: AI_RUN_ERROR_CODES.failed,
|
||||
});
|
||||
}
|
||||
case "error":
|
||||
// The check before the first read failed itself.
|
||||
return failure({
|
||||
status: 500,
|
||||
message: "The background read failed.",
|
||||
errorCode: AI_RUN_ERROR_CODES.failed,
|
||||
});
|
||||
default:
|
||||
return failure({
|
||||
status: 422,
|
||||
@@ -246,8 +251,16 @@ export async function executeOcrExtraction(args: {
|
||||
let reserved = budget?.reserved ?? 0;
|
||||
const reserveRead = budget
|
||||
? async () => {
|
||||
// The owner's cap; a reservation that names no known owner (the
|
||||
// params are stored JSON) stops the scan rather than taking the
|
||||
// larger cap.
|
||||
const cap =
|
||||
budget.owner === "operator" ? OPERATOR_COST_CAP : USER_PLAN_CAP;
|
||||
budget.owner === "operator"
|
||||
? OPERATOR_COST_CAP
|
||||
: budget.owner === "user"
|
||||
? USER_PLAN_CAP
|
||||
: null;
|
||||
if (cap === null) return false;
|
||||
const book = (tokens: number) =>
|
||||
reserveBudget(
|
||||
userId,
|
||||
@@ -259,7 +272,8 @@ export async function executeOcrExtraction(args: {
|
||||
undefined,
|
||||
{ countMessage: false },
|
||||
);
|
||||
const behind = Math.max(0, meter.spent - reserved);
|
||||
// Net of the cached share, as the settlement books it.
|
||||
const behind = Math.max(0, meter.spent - meter.cached - reserved);
|
||||
if (behind > 0) {
|
||||
const caughtUp = await book(behind);
|
||||
if (!caughtUp.allowed) return false;
|
||||
|
||||
@@ -142,10 +142,10 @@ export const ocrExtractResponse = z
|
||||
"Present only when part of a multi-part read produced nothing: the provider failed partway through, or a part's reply could not be read. The rows of the other parts are kept; these pages are not in `rows`. Empty when the parts were not pages (OCR'd text).",
|
||||
),
|
||||
stoppedBy: z
|
||||
.enum(["budget", "unavailable"])
|
||||
.enum(["budget", "unavailable", "error"])
|
||||
.optional()
|
||||
.describe(
|
||||
"Present only when the scan stopped before its last read for a reason other than the provider: `budget`, the day's AI budget ran out (every read after the first is reserved on its own); `unavailable`, AI was turned off for the record while it ran (the wire is re-checked before every read). The pages it did not reach are in `unreadPages`.",
|
||||
"Present only when the scan stopped before its last read for a reason other than the provider: `budget`, the day's AI budget ran out (every read after the first is reserved on its own); `unavailable`, AI was turned off for the record while it ran (the wire is re-checked before every read); `error`, that check or the reservation failed itself, so the scan stopped rather than read on unchecked. The pages it did not reach are in `unreadPages`.",
|
||||
),
|
||||
rowsCapped: z
|
||||
.object({
|
||||
@@ -160,7 +160,7 @@ export const ocrExtractResponse = z
|
||||
.meta({
|
||||
id: "OcrExtractResponse",
|
||||
description:
|
||||
"The proposed rows for the human review screen, served by `GET /api/ai-runs/{id}` for a `LABS_OCR_EXTRACT` run. A PDF of more than one page is read a page at a time, each rendered page sent with its text layer as a hint the image overrules, and the pages merged in order, a reading an earlier page already listed listed once. Never written to the person's record; the commit route is the only write. A run in which no read produced a single complete row fails with 422 `labs.ocr.outputLimit` when the reads stopped at the provider's output limit, else `labs.ocr.extractFailed`; OCR'd text too long for the reads the deadline allows is refused up front with 422 `labs.ocr.tooLarge`.",
|
||||
"The proposed rows for the human review screen, served by `GET /api/ai-runs/{id}` for a `LABS_OCR_EXTRACT` run. A PDF of more than one page is read a page at a time, each rendered page sent with its text layer as a hint the image overrules, and the pages merged in order, a reading an earlier page already listed listed once. Never written to the person's record; the commit route is the only write. A run in which no read produced a single complete row fails with 422 `labs.ocr.outputLimit` when the reads stopped at the provider's output limit, else `labs.ocr.extractFailed`; OCR'd text too long for the reads the deadline allows is refused up front with 422 `labs.ocr.tooLarge`. A scan stopped before any read produced a row answers with the stop: 429 `labs.ocr.budgetExceeded` when the day's budget refused a further read; the wire's own refusal (its status and code, e.g. 403 `consent.ai.required`, `assistant.disabled.*`, `module.disabled`) when AI was turned off for the record; 500 `aiRuns.failed` when the check before a read failed. A scan stopped after some rows succeeds with them and names the stop in `stoppedBy` (`budget`, `unavailable`, `error`) and the pages it did not reach in `unreadPages`.",
|
||||
});
|
||||
|
||||
const committedRow = z
|
||||
|
||||
@@ -222,10 +222,11 @@ export interface OcrExtractResponseDto {
|
||||
/**
|
||||
* Present only when the scan stopped before its last read for a reason
|
||||
* other than the provider: `budget`, the day's AI budget ran out;
|
||||
* `unavailable`, AI was turned off for the record while it ran. The pages
|
||||
* it did not reach are in `unreadPages`.
|
||||
* `unavailable`, AI was turned off for the record while it ran; `error`,
|
||||
* the check before a read failed, so the scan stopped rather than read on
|
||||
* unchecked. The pages it did not reach are in `unreadPages`.
|
||||
*/
|
||||
stoppedBy?: "budget" | "unavailable";
|
||||
stoppedBy?: "budget" | "unavailable" | "error";
|
||||
/**
|
||||
* Present only when the report held more readings than one scan carries
|
||||
* (`OCR_MAX_ROWS`): the first `kept` of `found` are in `rows`.
|
||||
|
||||
Reference in new issue
Block a user