fix(documents): keep an incomplete extraction on the document

A long document read in parts can end with a part cut at the provider's
output limit, a part never read, or more facts than one review takes.
That was told once, as a notice when the run finished; a review opened
later looked complete.

InboundDocument.extractionIncomplete (migration 0483) is written with
the facts by both stagings, the extract route or run and lab
auto-staging. The document detail publishes it and the review shows the
same note whenever it is opened. The disaster-recovery backup carries
it; an older file restores it as false.

Refs #1257

src/__tests__/baselines/source-function-lines.json

@@ -1,7 +1,7 @@
 {
   "$comment": "Production functions over 200 lines, keyed \"file :: name\", and their length. Maintained by src/__tests__/source-size-ratchet.test.ts.",
   "count": 350,
-  "total": 113407,
+  "total": 113403,
   "entries": {
     "app/achievements/page.tsx :: AchievementsPage": 206,
     "app/api/admin/backups/upload/route.ts :: apiHandler(arg)": 314,
@@ -94,7 +94,7 @@
     "components/documents/document-bulk-bar.tsx :: DocumentBulkBar": 204,
     "components/documents/document-card.tsx :: DocumentCard": 354,
     "components/documents/document-detail-sheet.tsx :: DocumentDetailSheet": 325,
-    "components/documents/document-facts-review.tsx :: DocumentFactsSection": 246,
+    "components/documents/document-facts-review.tsx :: DocumentFactsSection": 242,
     "components/documents/document-facts-review.tsx :: FactRow": 262,
     "components/documents/document-filter-bar.tsx :: DocumentFilterBar": 348,
     "components/documents/document-record-links.tsx :: DocumentRecordLinks": 202,

src/components/documents/__tests__/document-facts-review.test.tsx

@@ -97,6 +97,7 @@ function doc(
     summary: null,
     summaryGeneratedAt: null,
     summaryState: "NONE",
+    extractionIncomplete: false,
     ...overrides,
   };
 }
@@ -114,6 +115,19 @@ describe("<DocumentFactsSection>", () => {
     expect(html).toContain("Review extracted value");
   });
 
+  it("says the review may not be the whole document when the extraction was cut", () => {
+    const cut = render(
+      <DocumentFactsSection
+        doc={doc({ extractionIncomplete: true })}
+        {...baseProps}
+      />,
+    );
+    expect(cut).toContain('data-slot="document-facts-incomplete"');
+    expect(cut).toContain("could not be read in full");
+    const whole = render(<DocumentFactsSection doc={doc()} {...baseProps} />);
+    expect(whole).not.toContain('data-slot="document-facts-incomplete"');
+  });
+
   it("offers the stored-text extract for a read lab document with nothing staged", () => {
     const html = render(
       <DocumentFactsSection

src/components/documents/document-facts-review.tsx

@@ -493,6 +493,37 @@ function announceExtract(
   }
 }
 
+/** The review panel's title and how to use it. */
+function ReviewHeading() {
+  const { t } = useTranslations();
+  return (
+    <div className="space-y-1">
+      <p className="text-sm font-medium">{t("documents.review.title")}</p>
+      <p className="text-muted-foreground text-xs">
+        {t("documents.review.hint")}
+      </p>
+    </div>
+  );
+}
+
+/**
+ * The staged facts may leave part of the document out. Kept on the document
+ * by the extraction, so a review opened later says what the notice at the
+ * end of the run said.
+ */
+function ExtractionIncompleteNote() {
+  const { t } = useTranslations();
+  return (
+    <p
+      role="note"
+      data-slot="document-facts-incomplete"
+      className="text-foreground text-sm"
+    >
+      {t("documents.review.extractIncomplete")}
+    </p>
+  );
+}
+
 export function DocumentFactsSection({
   doc,
   canManage,
@@ -598,6 +629,9 @@ export function DocumentFactsSection({
 
   return (
     <div className="space-y-3" data-slot="document-facts-section">
+      {pending.length > 0 && doc.extractionIncomplete ? (
+        <ExtractionIncompleteNote />
+      ) : null}
       {pending.length > 0 ? (
         <>
           {!open ? (
@@ -617,14 +651,7 @@ export function DocumentFactsSection({
               data-slot="document-facts-review"
               className="border-border bg-muted/30 space-y-3 rounded-lg border p-3 md:p-4"
             >
-              <div className="space-y-1">
-                <p className="text-sm font-medium">
-                  {t("documents.review.title")}
-                </p>
-                <p className="text-muted-foreground text-xs">
-                  {t("documents.review.hint")}
-                </p>
-              </div>
+              <ReviewHeading />
               <ul className="space-y-2">
                 {pending.map((fact) => (
                   <FactRow

src/lib/documents/__tests__/auto-stage-labs.test.ts

@@ -173,6 +173,19 @@ describe("maybeAutoStageLabFacts", () => {
     expect(created.data.every((f) => f.status === "PENDING")).toBe(true);
   });
 
+  it("keeps on the document whether the staged facts may be missing a part", async () => {
+    await maybeAutoStageLabFacts("u1", "d1");
+    expect(updateMany.mock.calls[0]![0]).toMatchObject({
+      data: { extractionIncomplete: false },
+    });
+    const whole = await mockExtract.mock.results[0]!.value;
+    mockExtract.mockResolvedValueOnce({ ...whole, incomplete: true });
+    await maybeAutoStageLabFacts("u1", "d1");
+    expect(updateMany.mock.calls[1]![0]).toMatchObject({
+      data: { extractionIncomplete: true },
+    });
+  });
+
   it("plans a long report's reads against the run's deadline (#1274)", async () => {
     // Four hundred lines of a panel, many reads of text.
     const longReport = Array.from(

src/lib/documents/ai-runs/extract-run.ts

@@ -106,6 +106,7 @@ async function stageExtraction(
   documentId: string,
   userId: string,
   result: InboundExtractionResult,
+  incomplete: boolean,
 ) {
   return prisma.$transaction(async (tx) => {
     const locked = await tx.$queryRaw<{ status: string }[]>`
@@ -151,6 +152,9 @@ async function stageExtraction(
       data: {
         status: "EXTRACTED",
         providerType: result.providerType,
+        // Kept with the facts it describes, so a review opened later says
+        // what the notice at the end of the run said.
+        extractionIncomplete: incomplete,
         reportDate: result.reportDate
           ? dateOnlyAtNoonUtc(result.reportDate)
           : null,
@@ -320,10 +324,11 @@ export async function executeDocumentExtract(args: {
     return unreadableDocument(false);
   }
   await settle(true);
+  const incomplete = result.incomplete || plan.partial;
 
   let staged: StagedDocument;
   try {
-    staged = await stageExtraction(document.id, userId, result);
+    staged = await stageExtraction(document.id, userId, result, incomplete);
   } catch (err) {
     if (!(err instanceof StagingRefused)) {
       // The reservation is already settled for the call that was made; a
@@ -356,6 +361,6 @@ export async function executeDocumentExtract(args: {
   });
   return {
     ok: true,
-    data: { document: staged, incomplete: result.incomplete || plan.partial },
+    data: { document: staged, incomplete },
   };
 }

src/lib/documents/auto-stage-labs.ts

@@ -221,6 +221,7 @@ async function stageFactsIfStored(
   userId: string,
   documentId: string,
   result: InboundExtractionResult,
+  incomplete: boolean,
 ): Promise<number | null> {
   return prisma.$transaction(async (tx) => {
     const moved = await tx.inboundDocument.updateMany({
@@ -228,6 +229,7 @@ async function stageFactsIfStored(
       data: {
         status: "EXTRACTED",
         providerType: result.providerType,
+        extractionIncomplete: incomplete,
         reportDate: result.reportDate
           ? dateOnlyAtNoonUtc(result.reportDate)
           : null,
@@ -359,7 +361,15 @@ export async function maybeAutoStageLabFacts(
     return { staged: false, reason: "extract-failed" };
   }
 
-  const staged = await stageFactsIfStored(userId, documentId, result);
+  // Part of the document may be missing from the staged facts. Kept on the
+  // document, so the review says so whenever it is opened.
+  const incomplete = result.incomplete || plan.partial;
+  const staged = await stageFactsIfStored(
+    userId,
+    documentId,
+    result,
+    incomplete,
+  );
   if (staged === null) return { staged: false, reason: "raced" };
 
   annotate({
@@ -369,9 +379,7 @@ export async function maybeAutoStageLabFacts(
       facts: staged,
       provider: provider.pick.providerType,
       byKind: doc.kind === "LAB_RESULT",
-      // Part of the document may be missing from the staged facts; the
-      // manual extract says so to the person, this records it.
-      incomplete: result.incomplete || plan.partial,
+      incomplete,
     },
   });
   return { staged: true, facts: staged };

src/lib/documents/store.ts

@@ -377,5 +377,6 @@ export function serialiseDocumentDetail(
       ? doc.summaryGeneratedAt.toISOString()
       : null,
     summaryState,
+    extractionIncomplete: doc.extractionIncomplete,
   };
 }

src/lib/export/documents-backup-schema.ts

@@ -59,6 +59,7 @@ export const documentBackupSchema = z
       .nullable()
       .optional(),
     aiReadDeferred: z.boolean().optional(),
+    extractionIncomplete: z.boolean().optional(),
     summary: z.string().nullable().optional(),
     createdAt: isoDateTime.optional(),
     updatedAt: isoDateTime.optional(),

src/lib/export/documents-backup.ts

@@ -61,6 +61,8 @@ export interface DocumentBackupEntry {
   sourceInstance?: string | null;
   /** Canonical DR only: the import's hold on automatic AI reading. */
   aiReadDeferred?: boolean;
+  /** Canonical DR only: the staged facts may leave part of it out. */
+  extractionIncomplete?: boolean;
 }
 
 /** The document columns both purposes carry. */
@@ -96,6 +98,7 @@ const DOCUMENT_DR_METADATA_SELECT = {
   lastIndexAttemptAt: true,
   lastIndexOutcome: true,
   aiReadDeferred: true,
+  extractionIncomplete: true,
   updatedAt: true,
 } as const satisfies Prisma.InboundDocumentSelect;
 
@@ -166,6 +169,7 @@ function serialiseDocument(
       lastIndexAttemptAt: dr.lastIndexAttemptAt?.toISOString() ?? null,
       lastIndexOutcome: dr.lastIndexOutcome,
       aiReadDeferred: dr.aiReadDeferred,
+      extractionIncomplete: dr.extractionIncomplete,
     };
   }
 
@@ -404,6 +408,9 @@ export async function restoreDocuments(ctx: RestoreContext): Promise<void> {
         // An older file has no marker: those documents predate the hold and
         // restore as ordinary ones.
         aiReadDeferred: document.aiReadDeferred ?? false,
+        // An older file has no marker: its staged facts read as complete,
+        // as they were shown when it was written.
+        extractionIncomplete: document.extractionIncomplete ?? false,
         sourceSystem: document.sourceSystem ?? null,
         sourceId: document.sourceSystem ? (document.sourceId ?? null) : null,
         // A file written before v1.39.3 has no instance: the key then

src/lib/openapi/routes/documents.ts

@@ -328,6 +328,10 @@ const inboundDocumentDetail = inboundDocument
     summary: z.string().nullable(),
     summaryGeneratedAt: z.string().nullable(),
     summaryState: z.enum(DOCUMENT_SUMMARY_STATES),
+    extractionIncomplete: z.boolean().meta({
+      description:
+        "Whether the staged facts may leave part of the document out: a part of a long document was cut at the provider's output limit or not read, the text or pages ran past what the reads take, or the document held more facts than one review takes. Kept from the last extraction (the extract route, its background run, or automatic lab staging) until the next one replaces the facts; false for a document never extracted. A review shows it whenever it is opened, not only when the run ends.",
+    }),
   })
   .meta({
     id: "InboundDocumentDetail",

src/lib/validations/inbound-documents.ts

@@ -398,12 +398,12 @@ export interface InboundDocumentDetailDto extends InboundDocumentDto {
   /** When the background summary was generated (ISO 8601), or null. */
   summaryGeneratedAt: string | null;
   /**
-   * What became of the summary. A null `summary` is ambiguous on its own —
-   * never attempted, mid-flight, withheld by the safety screen and could-not-
-   * produce all look identical — so the view reads this instead of guessing
-   * "still generating".
+   * What became of the summary: a null `summary` alone cannot tell never
+   * attempted, mid-flight, withheld and could-not-produce apart.
    */
   summaryState: DocumentSummaryStateValue;
+  /** The staged facts may leave part of the document out (last extraction). */
+  extractionIncomplete: boolean;
 }
 
 // ─── Edit (correction before approval) ─────────────────────────────────────