mirror of
https://github.com/MBombeck/HealthLog.git
synced 2026-11-07 04:12:04 +00:00
fix(documents): keep an incomplete extraction on the document
A long document read in parts can end with a part cut at the provider's output limit, a part never read, or more facts than one review takes. That was told once, as a notice when the run finished; a review opened later looked complete. InboundDocument.extractionIncomplete (migration 0483) is written with the facts by both stagings, the extract route or run and lab auto-staging. The document detail publishes it and the review shows the same note whenever it is opened. The disaster-recovery backup carries it; an older file restores it as false. Refs #1257
This commit is contained in:
1 parent
5cfaf4869d
commit
958d230b9f
15 files changed
+233
-44
No files matched your search
@@ -43911,6 +43911,8 @@ components:
|
||||
- type: "null"
|
||||
aiReadDeferred:
|
||||
type: boolean
|
||||
extractionIncomplete:
|
||||
type: boolean
|
||||
summary:
|
||||
type:
|
||||
- string
|
||||
@@ -80342,6 +80344,13 @@ components:
|
||||
- READY
|
||||
- WITHHELD
|
||||
- UNAVAILABLE
|
||||
extractionIncomplete:
|
||||
type: boolean
|
||||
description: "Whether the staged facts may leave part of the document out: a part of a long document was cut at the
|
||||
provider's output limit or not read, the text or pages ran past what the reads take, or the document held
|
||||
more facts than one review takes. Kept from the last extraction (the extract route, its background run, or
|
||||
automatic lab staging) until the next one replaces the facts; false for a document never extracted. A review
|
||||
shows it whenever it is opened, not only when the run ends."
|
||||
required:
|
||||
- id
|
||||
- kind
|
||||
@@ -80373,6 +80382,7 @@ components:
|
||||
- summary
|
||||
- summaryGeneratedAt
|
||||
- summaryState
|
||||
- extractionIncomplete
|
||||
additionalProperties: false
|
||||
description: 'A stored document plus its staged facts. `summary` is a short (3-4 sentence) plain-language description of
|
||||
WHAT the document is, generated once and then served from storage; it is descriptive only, never a diagnosis.
|
||||
|
||||
@@ -0,0 +1,21 @@
|
||||
-- Whether a document's staged facts may leave part of it out.
|
||||
--
|
||||
-- An extraction that read a long document in parts can end with a part cut
|
||||
-- at the provider's output limit, a part never read, or more facts than one
|
||||
-- review takes. That was told once, as a notice when the run finished; a
|
||||
-- review opened later looked complete. The column keeps it on the document,
|
||||
-- written with the facts every staging replaces.
|
||||
--
|
||||
-- Not null, default false: a document extracted before this release is read
|
||||
-- as complete, which is what it has always been shown as.
|
||||
-- Reversibility: dropping the column loses only the marker.
|
||||
--
|
||||
-- @retry-on-lock-timeout
|
||||
|
||||
DO $migration$
|
||||
BEGIN
|
||||
PERFORM set_config('lock_timeout', '5s', true);
|
||||
ALTER TABLE "inbound_documents"
|
||||
ADD COLUMN IF NOT EXISTS "extraction_incomplete" BOOLEAN NOT NULL DEFAULT false;
|
||||
END
|
||||
$migration$;
|
||||
+32
-24
@@ -8896,9 +8896,9 @@ enum ExtractedFactStatus {
|
||||
}
|
||||
|
||||
model InboundDocument {
|
||||
id String @id @default(cuid())
|
||||
userId String @map("user_id")
|
||||
kind InboundDocumentKind @default(OTHER)
|
||||
id String @id @default(cuid())
|
||||
userId String @map("user_id")
|
||||
kind InboundDocumentKind @default(OTHER)
|
||||
/// User-given label. Deliberately plaintext (like `filename`) so the library
|
||||
/// list can ILIKE-search + ORDER BY in SQL. The document BODY stays
|
||||
/// encrypted; only this short, user-authored title is clear-text. Tradeoff:
|
||||
@@ -8906,18 +8906,18 @@ model InboundDocument {
|
||||
/// cleartext at rest / in backups — the same exposure the pre-existing
|
||||
/// plaintext `filename` already carries. Accepted to keep server-side search
|
||||
/// and sort.
|
||||
title String?
|
||||
title String?
|
||||
/// Original filename as uploaded (sanitised, optional).
|
||||
filename String?
|
||||
filename String?
|
||||
/// Magic-byte-sniffed MIME type the document was stored as.
|
||||
mimeType String @map("mime_type")
|
||||
mimeType String @map("mime_type")
|
||||
/// Stored byte length of the original (pre-encryption) document.
|
||||
byteSize Int @map("byte_size")
|
||||
byteSize Int @map("byte_size")
|
||||
/// AES-256-GCM ciphertext of the raw document bytes (base64 of the binary
|
||||
/// → `encrypt()` string → UTF-8 BYTEA), the `*_encrypted` Bytes convention
|
||||
/// shared with the note columns. The original is sensitive and never
|
||||
/// stored in the clear or logged.
|
||||
contentEncrypted Bytes @map("content_encrypted")
|
||||
contentEncrypted Bytes @map("content_encrypted")
|
||||
/// Lowercase-hex SHA-256 of the PLAINTEXT document bytes, computed at
|
||||
/// upload for same-user duplicate detection. Nullable: rows stored before
|
||||
/// the vault release are not backfilled (only new uploads dedupe). A
|
||||
@@ -8926,29 +8926,29 @@ model InboundDocument {
|
||||
/// CREATE UNIQUE INDEX inbound_documents_user_sha_live
|
||||
/// ON inbound_documents (user_id, content_sha256)
|
||||
/// WHERE deleted_at IS NULL AND content_sha256 IS NOT NULL;
|
||||
contentSha256 String? @map("content_sha256") @db.VarChar(64)
|
||||
contentSha256 String? @map("content_sha256") @db.VarChar(64)
|
||||
/// Storage codec of `contentEncrypted`. "base64v1" = legacy string path
|
||||
/// (base64-of-binary → `encrypt()` string → UTF-8 bytes); "binary2" = the
|
||||
/// binary AES-256-GCM layout written by `encryptBytes()` (no base64
|
||||
/// detour). Explicit column, never header-sniffed; the default keeps every
|
||||
/// pre-existing row correct.
|
||||
contentCodec String @default("base64v1") @map("content_codec")
|
||||
status InboundDocumentStatus @default(STORED)
|
||||
contentCodec String @default("base64v1") @map("content_codec")
|
||||
status InboundDocumentStatus @default(STORED)
|
||||
/// Provider type that ran the extraction (e.g. "anthropic", "openai",
|
||||
/// "local"). Diagnostic only. Null on a store-only document.
|
||||
providerType String? @map("provider_type")
|
||||
providerType String? @map("provider_type")
|
||||
/// The document's stated report/collection date (YYYY-MM-DD at UTC), as
|
||||
/// transcribed by the model — never inferred, never user-set.
|
||||
reportDate DateTime? @map("report_date")
|
||||
reportDate DateTime? @map("report_date")
|
||||
/// Effective filing date the library sorts, groups, and range-filters on
|
||||
/// (editable). Distinct from `reportDate`, which is the model-transcribed
|
||||
/// value. On store it defaults to the upload day (now) when the user does
|
||||
/// not supply one, so a stored row's filing date is never NULL and display
|
||||
/// == sort == filter. It stays user-editable and MAY be cleared back to
|
||||
/// NULL via PATCH (the list then sorts such rows last).
|
||||
documentDate DateTime? @map("document_date")
|
||||
documentDate DateTime? @map("document_date")
|
||||
/// Failure reason when status = FAILED (a short code, never the document).
|
||||
errorReason String? @map("error_reason")
|
||||
errorReason String? @map("error_reason")
|
||||
/// AES-256-GCM ciphertext of a short (3-4 sentence) plain-language summary
|
||||
/// of WHAT the document is, generated ONCE in the background right after
|
||||
/// upload when the user's `documentsAutoAiRead` opt-in is ON (that flag is
|
||||
@@ -8956,23 +8956,23 @@ model InboundDocument {
|
||||
/// provider is configured, or generation has not run yet. Descriptive only —
|
||||
/// never a diagnosis. `encrypt()`-string-as-UTF-8 BYTEA, the shared Bytes
|
||||
/// codec every other note column uses; PHI, never logged.
|
||||
summaryEncrypted Bytes? @map("summary_encrypted")
|
||||
summaryEncrypted Bytes? @map("summary_encrypted")
|
||||
/// When the background summary was generated (null until it runs). Surfaced
|
||||
/// as meta beside the summary in the document detail view.
|
||||
summaryGeneratedAt DateTime? @map("summary_generated_at")
|
||||
summaryGeneratedAt DateTime? @map("summary_generated_at")
|
||||
/// What actually happened to this document's summary, so the detail view can
|
||||
/// tell the four outcomes apart instead of reading a null column as "still
|
||||
/// generating". `summaryEncrypted` alone cannot: a document nobody ever
|
||||
/// enqueued, one whose provider is missing, one the safety screen withheld,
|
||||
/// and one mid-flight all leave it null. NONE is the default, which keeps
|
||||
/// every pre-existing row correct — none of them were ever attempted.
|
||||
summaryState DocumentSummaryState @default(NONE) @map("summary_state")
|
||||
summaryState DocumentSummaryState @default(NONE) @map("summary_state")
|
||||
/// When the most recent content-index attempt finished (auto job, corpus
|
||||
/// backfill, or the manual index route — they share one recorder),
|
||||
/// successful or not. Null until any attempt ran. Diagnostic state the
|
||||
/// detail view pairs with `lastIndexOutcome`; carried verbatim in the
|
||||
/// disaster-recovery backup like its summary siblings.
|
||||
lastIndexAttemptAt DateTime? @map("last_index_attempt_at")
|
||||
lastIndexAttemptAt DateTime? @map("last_index_attempt_at")
|
||||
/// Why the most recent index attempt produced NO index — a closed string
|
||||
/// enum (`DocumentIndexOutcomeValue` in
|
||||
/// `src/lib/validations/inbound-documents.ts`): local-empty |
|
||||
@@ -8982,13 +8982,13 @@ model InboundDocument {
|
||||
/// and before the first attempt. Lets the detail view tell "never tried"
|
||||
/// from "tried and here is why it failed" instead of showing a bare
|
||||
/// not-indexed state for both.
|
||||
lastIndexOutcome String? @map("last_index_outcome")
|
||||
lastIndexOutcome String? @map("last_index_outcome")
|
||||
/// v1.39.2 (#1038) — where an imported document came from: a closed string
|
||||
/// enum (`DOCUMENT_SOURCE_SYSTEMS` in
|
||||
/// `src/lib/validations/inbound-documents.ts`): PAPERLESS | PAPRA | OTHER.
|
||||
/// Null for a document added in HealthLog itself. Sent by the import script
|
||||
/// and the Paperless workflow recipe; shown on the detail sheet.
|
||||
sourceSystem String? @map("source_system") @db.VarChar(16)
|
||||
sourceSystem String? @map("source_system") @db.VarChar(16)
|
||||
/// v1.39.2 (#1038) — the document's id in `sourceSystem`. Together with
|
||||
/// `userId`, `sourceSystem` and `sourceInstance` it is unique across live
|
||||
/// AND tombstoned rows (raw SQL in migrations 0353 and 0356, not expressible
|
||||
@@ -8996,7 +8996,7 @@ model InboundDocument {
|
||||
/// Including tombstones is the point: re-running an importer must not bring
|
||||
/// back a document the person deleted. The purge carries the key over to
|
||||
/// `DocumentImportKey` when it removes the row.
|
||||
sourceId String? @map("source_id") @db.VarChar(128)
|
||||
sourceId String? @map("source_id") @db.VarChar(128)
|
||||
/// v1.39.3 (#1038) — which instance of `sourceSystem` the document came
|
||||
/// from: the normalised origin (`scheme://host[:port]`) of the Paperless-ngx
|
||||
/// or Papra it was fetched from, so id 123 on one Paperless and id 123 on
|
||||
@@ -9007,12 +9007,20 @@ model InboundDocument {
|
||||
/// source-key unique index since migration 0356, which rebuilt it as
|
||||
/// (user_id, source_system, source_instance, source_id) NULLS NOT DISTINCT
|
||||
/// WHERE source_id IS NOT NULL.
|
||||
sourceInstance String? @map("source_instance") @db.VarChar(512)
|
||||
sourceInstance String? @map("source_instance") @db.VarChar(512)
|
||||
/// v1.39.2 (#1038) — the upload asked for `aiRead=defer`: nothing may send
|
||||
/// this document to an AI provider on its own (the summary catch-up, the
|
||||
/// summary job, lab auto-staging all skip it). Cleared when the person
|
||||
/// explicitly reads it with AI or generates its summary.
|
||||
aiReadDeferred Boolean @default(false) @map("ai_read_deferred")
|
||||
aiReadDeferred Boolean @default(false) @map("ai_read_deferred")
|
||||
/// Whether the facts staged by the last extraction may leave part of the
|
||||
/// document out: a read stopped at the provider's output limit, came back
|
||||
/// unreadable or was never sent, the text or pages ran past what the reads
|
||||
/// take, or the document held more facts than one review takes. Written by
|
||||
/// both stagings (the extract route or run, and lab auto-staging) with the
|
||||
/// facts they replace, so a review opened later says so, not only the
|
||||
/// notice at the end of the run. False for a document never extracted.
|
||||
extractionIncomplete Boolean @default(false) @map("extraction_incomplete")
|
||||
|
||||
facts ExtractedFact[]
|
||||
conditionLinks DocumentConditionLink[]
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
{
|
||||
"$comment": "Production functions over 200 lines, keyed \"file :: name\", and their length. Maintained by src/__tests__/source-size-ratchet.test.ts.",
|
||||
"count": 350,
|
||||
"total": 113407,
|
||||
"total": 113403,
|
||||
"entries": {
|
||||
"app/achievements/page.tsx :: AchievementsPage": 206,
|
||||
"app/api/admin/backups/upload/route.ts :: apiHandler(arg)": 314,
|
||||
@@ -94,7 +94,7 @@
|
||||
"components/documents/document-bulk-bar.tsx :: DocumentBulkBar": 204,
|
||||
"components/documents/document-card.tsx :: DocumentCard": 354,
|
||||
"components/documents/document-detail-sheet.tsx :: DocumentDetailSheet": 325,
|
||||
"components/documents/document-facts-review.tsx :: DocumentFactsSection": 246,
|
||||
"components/documents/document-facts-review.tsx :: DocumentFactsSection": 242,
|
||||
"components/documents/document-facts-review.tsx :: FactRow": 262,
|
||||
"components/documents/document-filter-bar.tsx :: DocumentFilterBar": 348,
|
||||
"components/documents/document-record-links.tsx :: DocumentRecordLinks": 202,
|
||||
|
||||
@@ -97,6 +97,7 @@ function doc(
|
||||
summary: null,
|
||||
summaryGeneratedAt: null,
|
||||
summaryState: "NONE",
|
||||
extractionIncomplete: false,
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
@@ -114,6 +115,19 @@ describe("<DocumentFactsSection>", () => {
|
||||
expect(html).toContain("Review extracted value");
|
||||
});
|
||||
|
||||
it("says the review may not be the whole document when the extraction was cut", () => {
|
||||
const cut = render(
|
||||
<DocumentFactsSection
|
||||
doc={doc({ extractionIncomplete: true })}
|
||||
{...baseProps}
|
||||
/>,
|
||||
);
|
||||
expect(cut).toContain('data-slot="document-facts-incomplete"');
|
||||
expect(cut).toContain("could not be read in full");
|
||||
const whole = render(<DocumentFactsSection doc={doc()} {...baseProps} />);
|
||||
expect(whole).not.toContain('data-slot="document-facts-incomplete"');
|
||||
});
|
||||
|
||||
it("offers the stored-text extract for a read lab document with nothing staged", () => {
|
||||
const html = render(
|
||||
<DocumentFactsSection
|
||||
|
||||
@@ -493,6 +493,37 @@ function announceExtract(
|
||||
}
|
||||
}
|
||||
|
||||
/** The review panel's title and how to use it. */
|
||||
function ReviewHeading() {
|
||||
const { t } = useTranslations();
|
||||
return (
|
||||
<div className="space-y-1">
|
||||
<p className="text-sm font-medium">{t("documents.review.title")}</p>
|
||||
<p className="text-muted-foreground text-xs">
|
||||
{t("documents.review.hint")}
|
||||
</p>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* The staged facts may leave part of the document out. Kept on the document
|
||||
* by the extraction, so a review opened later says what the notice at the
|
||||
* end of the run said.
|
||||
*/
|
||||
function ExtractionIncompleteNote() {
|
||||
const { t } = useTranslations();
|
||||
return (
|
||||
<p
|
||||
role="note"
|
||||
data-slot="document-facts-incomplete"
|
||||
className="text-foreground text-sm"
|
||||
>
|
||||
{t("documents.review.extractIncomplete")}
|
||||
</p>
|
||||
);
|
||||
}
|
||||
|
||||
export function DocumentFactsSection({
|
||||
doc,
|
||||
canManage,
|
||||
@@ -598,6 +629,9 @@ export function DocumentFactsSection({
|
||||
|
||||
return (
|
||||
<div className="space-y-3" data-slot="document-facts-section">
|
||||
{pending.length > 0 && doc.extractionIncomplete ? (
|
||||
<ExtractionIncompleteNote />
|
||||
) : null}
|
||||
{pending.length > 0 ? (
|
||||
<>
|
||||
{!open ? (
|
||||
@@ -617,14 +651,7 @@ export function DocumentFactsSection({
|
||||
data-slot="document-facts-review"
|
||||
className="border-border bg-muted/30 space-y-3 rounded-lg border p-3 md:p-4"
|
||||
>
|
||||
<div className="space-y-1">
|
||||
<p className="text-sm font-medium">
|
||||
{t("documents.review.title")}
|
||||
</p>
|
||||
<p className="text-muted-foreground text-xs">
|
||||
{t("documents.review.hint")}
|
||||
</p>
|
||||
</div>
|
||||
<ReviewHeading />
|
||||
<ul className="space-y-2">
|
||||
{pending.map((fact) => (
|
||||
<FactRow
|
||||
|
||||
@@ -173,6 +173,19 @@ describe("maybeAutoStageLabFacts", () => {
|
||||
expect(created.data.every((f) => f.status === "PENDING")).toBe(true);
|
||||
});
|
||||
|
||||
it("keeps on the document whether the staged facts may be missing a part", async () => {
|
||||
await maybeAutoStageLabFacts("u1", "d1");
|
||||
expect(updateMany.mock.calls[0]![0]).toMatchObject({
|
||||
data: { extractionIncomplete: false },
|
||||
});
|
||||
const whole = await mockExtract.mock.results[0]!.value;
|
||||
mockExtract.mockResolvedValueOnce({ ...whole, incomplete: true });
|
||||
await maybeAutoStageLabFacts("u1", "d1");
|
||||
expect(updateMany.mock.calls[1]![0]).toMatchObject({
|
||||
data: { extractionIncomplete: true },
|
||||
});
|
||||
});
|
||||
|
||||
it("plans a long report's reads against the run's deadline (#1274)", async () => {
|
||||
// Four hundred lines of a panel, many reads of text.
|
||||
const longReport = Array.from(
|
||||
|
||||
@@ -106,6 +106,7 @@ async function stageExtraction(
|
||||
documentId: string,
|
||||
userId: string,
|
||||
result: InboundExtractionResult,
|
||||
incomplete: boolean,
|
||||
) {
|
||||
return prisma.$transaction(async (tx) => {
|
||||
const locked = await tx.$queryRaw<{ status: string }[]>`
|
||||
@@ -151,6 +152,9 @@ async function stageExtraction(
|
||||
data: {
|
||||
status: "EXTRACTED",
|
||||
providerType: result.providerType,
|
||||
// Kept with the facts it describes, so a review opened later says
|
||||
// what the notice at the end of the run said.
|
||||
extractionIncomplete: incomplete,
|
||||
reportDate: result.reportDate
|
||||
? dateOnlyAtNoonUtc(result.reportDate)
|
||||
: null,
|
||||
@@ -320,10 +324,11 @@ export async function executeDocumentExtract(args: {
|
||||
return unreadableDocument(false);
|
||||
}
|
||||
await settle(true);
|
||||
const incomplete = result.incomplete || plan.partial;
|
||||
|
||||
let staged: StagedDocument;
|
||||
try {
|
||||
staged = await stageExtraction(document.id, userId, result);
|
||||
staged = await stageExtraction(document.id, userId, result, incomplete);
|
||||
} catch (err) {
|
||||
if (!(err instanceof StagingRefused)) {
|
||||
// The reservation is already settled for the call that was made; a
|
||||
@@ -356,6 +361,6 @@ export async function executeDocumentExtract(args: {
|
||||
});
|
||||
return {
|
||||
ok: true,
|
||||
data: { document: staged, incomplete: result.incomplete || plan.partial },
|
||||
data: { document: staged, incomplete },
|
||||
};
|
||||
}
|
||||
@@ -221,6 +221,7 @@ async function stageFactsIfStored(
|
||||
userId: string,
|
||||
documentId: string,
|
||||
result: InboundExtractionResult,
|
||||
incomplete: boolean,
|
||||
): Promise<number | null> {
|
||||
return prisma.$transaction(async (tx) => {
|
||||
const moved = await tx.inboundDocument.updateMany({
|
||||
@@ -228,6 +229,7 @@ async function stageFactsIfStored(
|
||||
data: {
|
||||
status: "EXTRACTED",
|
||||
providerType: result.providerType,
|
||||
extractionIncomplete: incomplete,
|
||||
reportDate: result.reportDate
|
||||
? dateOnlyAtNoonUtc(result.reportDate)
|
||||
: null,
|
||||
@@ -359,7 +361,15 @@ export async function maybeAutoStageLabFacts(
|
||||
return { staged: false, reason: "extract-failed" };
|
||||
}
|
||||
|
||||
const staged = await stageFactsIfStored(userId, documentId, result);
|
||||
// Part of the document may be missing from the staged facts. Kept on the
|
||||
// document, so the review says so whenever it is opened.
|
||||
const incomplete = result.incomplete || plan.partial;
|
||||
const staged = await stageFactsIfStored(
|
||||
userId,
|
||||
documentId,
|
||||
result,
|
||||
incomplete,
|
||||
);
|
||||
if (staged === null) return { staged: false, reason: "raced" };
|
||||
|
||||
annotate({
|
||||
@@ -369,9 +379,7 @@ export async function maybeAutoStageLabFacts(
|
||||
facts: staged,
|
||||
provider: provider.pick.providerType,
|
||||
byKind: doc.kind === "LAB_RESULT",
|
||||
// Part of the document may be missing from the staged facts; the
|
||||
// manual extract says so to the person, this records it.
|
||||
incomplete: result.incomplete || plan.partial,
|
||||
incomplete,
|
||||
},
|
||||
});
|
||||
return { staged: true, facts: staged };
|
||||
|
||||
@@ -377,5 +377,6 @@ export function serialiseDocumentDetail(
|
||||
? doc.summaryGeneratedAt.toISOString()
|
||||
: null,
|
||||
summaryState,
|
||||
extractionIncomplete: doc.extractionIncomplete,
|
||||
};
|
||||
}
|
||||
@@ -59,6 +59,7 @@ export const documentBackupSchema = z
|
||||
.nullable()
|
||||
.optional(),
|
||||
aiReadDeferred: z.boolean().optional(),
|
||||
extractionIncomplete: z.boolean().optional(),
|
||||
summary: z.string().nullable().optional(),
|
||||
createdAt: isoDateTime.optional(),
|
||||
updatedAt: isoDateTime.optional(),
|
||||
|
||||
@@ -61,6 +61,8 @@ export interface DocumentBackupEntry {
|
||||
sourceInstance?: string | null;
|
||||
/** Canonical DR only: the import's hold on automatic AI reading. */
|
||||
aiReadDeferred?: boolean;
|
||||
/** Canonical DR only: the staged facts may leave part of it out. */
|
||||
extractionIncomplete?: boolean;
|
||||
}
|
||||
|
||||
/** The document columns both purposes carry. */
|
||||
@@ -96,6 +98,7 @@ const DOCUMENT_DR_METADATA_SELECT = {
|
||||
lastIndexAttemptAt: true,
|
||||
lastIndexOutcome: true,
|
||||
aiReadDeferred: true,
|
||||
extractionIncomplete: true,
|
||||
updatedAt: true,
|
||||
} as const satisfies Prisma.InboundDocumentSelect;
|
||||
|
||||
@@ -166,6 +169,7 @@ function serialiseDocument(
|
||||
lastIndexAttemptAt: dr.lastIndexAttemptAt?.toISOString() ?? null,
|
||||
lastIndexOutcome: dr.lastIndexOutcome,
|
||||
aiReadDeferred: dr.aiReadDeferred,
|
||||
extractionIncomplete: dr.extractionIncomplete,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -404,6 +408,9 @@ export async function restoreDocuments(ctx: RestoreContext): Promise<void> {
|
||||
// An older file has no marker: those documents predate the hold and
|
||||
// restore as ordinary ones.
|
||||
aiReadDeferred: document.aiReadDeferred ?? false,
|
||||
// An older file has no marker: its staged facts read as complete,
|
||||
// as they were shown when it was written.
|
||||
extractionIncomplete: document.extractionIncomplete ?? false,
|
||||
sourceSystem: document.sourceSystem ?? null,
|
||||
sourceId: document.sourceSystem ? (document.sourceId ?? null) : null,
|
||||
// A file written before v1.39.3 has no instance: the key then
|
||||
|
||||
@@ -328,6 +328,10 @@ const inboundDocumentDetail = inboundDocument
|
||||
summary: z.string().nullable(),
|
||||
summaryGeneratedAt: z.string().nullable(),
|
||||
summaryState: z.enum(DOCUMENT_SUMMARY_STATES),
|
||||
extractionIncomplete: z.boolean().meta({
|
||||
description:
|
||||
"Whether the staged facts may leave part of the document out: a part of a long document was cut at the provider's output limit or not read, the text or pages ran past what the reads take, or the document held more facts than one review takes. Kept from the last extraction (the extract route, its background run, or automatic lab staging) until the next one replaces the facts; false for a document never extracted. A review shows it whenever it is opened, not only when the run ends.",
|
||||
}),
|
||||
})
|
||||
.meta({
|
||||
id: "InboundDocumentDetail",
|
||||
|
||||
@@ -398,12 +398,12 @@ export interface InboundDocumentDetailDto extends InboundDocumentDto {
|
||||
/** When the background summary was generated (ISO 8601), or null. */
|
||||
summaryGeneratedAt: string | null;
|
||||
/**
|
||||
* What became of the summary. A null `summary` is ambiguous on its own —
|
||||
* never attempted, mid-flight, withheld by the safety screen and could-not-
|
||||
* produce all look identical — so the view reads this instead of guessing
|
||||
* "still generating".
|
||||
* What became of the summary: a null `summary` alone cannot tell never
|
||||
* attempted, mid-flight, withheld and could-not-produce apart.
|
||||
*/
|
||||
summaryState: DocumentSummaryStateValue;
|
||||
/** The staged facts may leave part of the document out (last extraction). */
|
||||
extractionIncomplete: boolean;
|
||||
}
|
||||
|
||||
// ─── Edit (correction before approval) ─────────────────────────────────────
|
||||
|
||||
@@ -854,3 +854,73 @@ describe("fact extraction in the background", () => {
|
||||
expect(await getPrismaClient().documentAiRun.count()).toBe(0);
|
||||
});
|
||||
});
|
||||
|
||||
async function documentDetail(documentId: string) {
|
||||
const { GET } = await import("@/app/api/documents/inbound/[id]/route");
|
||||
const { NextRequest } = await import("next/server");
|
||||
const res = await GET(
|
||||
new NextRequest(
|
||||
`http://localhost/api/documents/inbound/${documentId}`,
|
||||
) as never,
|
||||
{ params: Promise.resolve({ id: documentId }) } as never,
|
||||
);
|
||||
return {
|
||||
status: res.status,
|
||||
body: (await res.json()) as Envelope<Record<string, unknown>>,
|
||||
};
|
||||
}
|
||||
|
||||
/** The extraction mock's default answer, cut short. */
|
||||
async function answerIncompleteOnce() {
|
||||
const actual = await vi.mocked(runInboundExtraction).getMockImplementation()!(
|
||||
{} as never,
|
||||
);
|
||||
vi.mocked(runInboundExtraction).mockResolvedValueOnce({
|
||||
...actual,
|
||||
incomplete: true,
|
||||
});
|
||||
}
|
||||
|
||||
describe("an extraction that did not read the whole document", () => {
|
||||
it("keeps saying so on the document until a whole read replaces it", async () => {
|
||||
const user = await makeUser();
|
||||
await grantConsent(user.id);
|
||||
const document = await indexedDocument(user.id);
|
||||
await signIn(user.id);
|
||||
|
||||
await answerIncompleteOnce();
|
||||
const cut = await postStoredExtract(document.id, false);
|
||||
expect(cut.status).toBe(200);
|
||||
expect(cut.body.data).toMatchObject({ extractionIncomplete: true });
|
||||
// Opened later, the review still says it is not the whole document.
|
||||
const later = await documentDetail(document.id);
|
||||
expect(later.status).toBe(200);
|
||||
expect(later.body.data).toMatchObject({ extractionIncomplete: true });
|
||||
|
||||
const whole = await postStoredExtract(document.id, false);
|
||||
expect(whole.status).toBe(200);
|
||||
expect(whole.body.data).toMatchObject({ extractionIncomplete: false });
|
||||
expect((await documentDetail(document.id)).body.data).toMatchObject({
|
||||
extractionIncomplete: false,
|
||||
});
|
||||
});
|
||||
|
||||
it("records it from a background run as well", async () => {
|
||||
const user = await makeUser();
|
||||
await grantConsent(user.id);
|
||||
const document = await indexedDocument(user.id);
|
||||
await signIn(user.id);
|
||||
|
||||
await answerIncompleteOnce();
|
||||
const queued = await postStoredExtract(document.id, true);
|
||||
expect(queued.status).toBe(202);
|
||||
const { runId } = queued.body.data as { runId: string };
|
||||
expect(await work(runId)).toBe("succeeded");
|
||||
expect((await poll(runId)).body.data).toMatchObject({
|
||||
result: { incomplete: true },
|
||||
});
|
||||
expect((await documentDetail(document.id)).body.data).toMatchObject({
|
||||
extractionIncomplete: true,
|
||||
});
|
||||
});
|
||||
});
|
||||
Reference in new issue
Block a user