From 5cff7473c9cf623d1721e3ad5f270dcee0057a17 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Fri, 17 Jul 2026 17:46:28 +0200 Subject: [PATCH] feat(documents): persist an auto summary for uploaded documents When the auto-AI-read opt-in is on, a freshly uploaded document is summarised once in the background and the encrypted summary is stored and shown on the document detail view. - A background step runs after upload alongside the existing content-index and thumbnail steps. It generates a short (3-4 sentence) plain-language summary only when the user's documentsAutoAiRead opt-in is on (that flag is also the standing AI-egress consent), skips a document that already has one, and no-ops when no vision provider is configured. It reuses the same provider resolution, egress-consent re-assertion, and AI-budget reserve/reconcile helpers as the on-demand route; a provider error refunds the reservation and never fails the upload. - The summary is stored encrypted (summaryEncrypted Bytes) with a summaryGeneratedAt timestamp on inbound_documents, registered for key rotation, and returned decrypted in the document detail response. - The on-demand transient summary route is unchanged. Gated by the same consent, budget, and module checks. --- docs/api/openapi.yaml | 15 ++ messages/de.json | 4 + messages/en.json | 4 + messages/es.json | 4 + messages/fr.json | 4 + messages/it.json | 4 + messages/pl.json | 4 + .../migration.sql | 23 ++ prisma/schema.prisma | 51 ++-- scripts/rotate-encryption-key.ts | 12 + src/app/api/documents/inbound/route.ts | 8 + .../documents/document-detail-sheet.tsx | 29 +++ src/lib/crypto/encrypted-columns.ts | 6 + src/lib/documents/store.ts | 35 +++ .../jobs/__tests__/document-summary.test.ts | 191 ++++++++++++++ src/lib/jobs/document-summary.ts | 243 ++++++++++++++++++ src/lib/jobs/reminder/register-maintenance.ts | 38 +++ src/lib/openapi/routes/documents.ts | 12 +- src/lib/validations/inbound-documents.ts | 9 + 19 files changed, 674 insertions(+), 22 deletions(-) create mode 100644 prisma/migrations/0250_inbound_document_summary/migration.sql create mode 100644 src/lib/jobs/__tests__/document-summary.test.ts create mode 100644 src/lib/jobs/document-summary.ts diff --git a/docs/api/openapi.yaml b/docs/api/openapi.yaml index db9fade75..e67dddb18 100644 --- a/docs/api/openapi.yaml +++ b/docs/api/openapi.yaml @@ -29085,6 +29085,14 @@ components: type: array items: $ref: "#/components/schemas/ExtractedFact" + summary: + anyOf: + - type: string + - type: "null" + summaryGeneratedAt: + anyOf: + - type: string + - type: "null" required: - id - kind @@ -29107,7 +29115,14 @@ components: - createdAt - updatedAt - facts + - summary + - summaryGeneratedAt additionalProperties: false + description: A stored document plus its staged facts. `summary` is a short (3-4 sentence) plain-language description of + WHAT the document is, generated once in the background right after upload when the `documentsAutoAiRead` opt-in + is ON (that flag is also the standing AI-egress consent); it is descriptive only, never a diagnosis, and is null + when auto-read is OFF, no provider is configured, or the background summary has not run yet. + `summaryGeneratedAt` is when it was generated (null until then). ExtractedFact: type: object properties: diff --git a/messages/de.json b/messages/de.json index d37ac34c6..ebbbc4a2f 100644 --- a/messages/de.json +++ b/messages/de.json @@ -8331,6 +8331,10 @@ }, "detail": { "title": "Dokumentdetails", + "summary": { + "title": "Zusammenfassung", + "pending": "Die Zusammenfassung wird erstellt." + }, "loadError": "Dieses Dokument konnte nicht geladen werden.", "saveError": "Die Änderung konnte nicht gespeichert werden.", "deleteError": "Das Dokument konnte nicht gelöscht werden.", diff --git a/messages/en.json b/messages/en.json index e6046c31f..99150d87f 100644 --- a/messages/en.json +++ b/messages/en.json @@ -8331,6 +8331,10 @@ }, "detail": { "title": "Document details", + "summary": { + "title": "Summary", + "pending": "The summary is being prepared." + }, "loadError": "Couldn't load this document.", "saveError": "Couldn't save the change.", "deleteError": "Couldn't delete the document.", diff --git a/messages/es.json b/messages/es.json index 78dc73526..20a61cc56 100644 --- a/messages/es.json +++ b/messages/es.json @@ -8331,6 +8331,10 @@ }, "detail": { "title": "Detalles del documento", + "summary": { + "title": "Resumen", + "pending": "Se está preparando el resumen." + }, "loadError": "No se pudo cargar este documento.", "saveError": "No se pudo guardar el cambio.", "deleteError": "No se pudo eliminar el documento.", diff --git a/messages/fr.json b/messages/fr.json index 26b85adea..06ac7ded4 100644 --- a/messages/fr.json +++ b/messages/fr.json @@ -8331,6 +8331,10 @@ }, "detail": { "title": "Détails du document", + "summary": { + "title": "Résumé", + "pending": "Le résumé est en cours de préparation." + }, "loadError": "Impossible de charger ce document.", "saveError": "Impossible d'enregistrer la modification.", "deleteError": "Impossible de supprimer le document.", diff --git a/messages/it.json b/messages/it.json index 0b879650e..b7ae23b87 100644 --- a/messages/it.json +++ b/messages/it.json @@ -8331,6 +8331,10 @@ }, "detail": { "title": "Dettagli del documento", + "summary": { + "title": "Riepilogo", + "pending": "Il riepilogo è in preparazione." + }, "loadError": "Impossibile caricare questo documento.", "saveError": "Impossibile salvare la modifica.", "deleteError": "Impossibile eliminare il documento.", diff --git a/messages/pl.json b/messages/pl.json index 7c09659ab..8ea4512af 100644 --- a/messages/pl.json +++ b/messages/pl.json @@ -8331,6 +8331,10 @@ }, "detail": { "title": "Szczegóły dokumentu", + "summary": { + "title": "Podsumowanie", + "pending": "Podsumowanie jest przygotowywane." + }, "loadError": "Nie udało się wczytać tego dokumentu.", "saveError": "Nie udało się zapisać zmiany.", "deleteError": "Nie udało się usunąć dokumentu.", diff --git a/prisma/migrations/0250_inbound_document_summary/migration.sql b/prisma/migrations/0250_inbound_document_summary/migration.sql new file mode 100644 index 000000000..d2e320dfc --- /dev/null +++ b/prisma/migrations/0250_inbound_document_summary/migration.sql @@ -0,0 +1,23 @@ +-- Persist a short plain-language summary of an uploaded document. +-- +-- Adds two nullable columns to `inbound_documents`: +-- - `summary_encrypted` — AES-256-GCM ciphertext of a 3-4 sentence +-- plain-language description of WHAT the document is, generated ONCE in +-- the background right after upload when the user's `documents_auto_ai_read` +-- opt-in is ON (that flag is also the standing AI-egress consent). NULL +-- when auto-read is OFF, no provider is configured, or generation has not +-- run yet. +-- - `summary_generated_at` — when that background summary was generated +-- (NULL until it runs). +-- +-- STORAGE CONVENTION: Bytes (`bytea`), matching the dominant free-text +-- encrypted-note convention already in the schema (the `encrypt()` ciphertext +-- string stored UTF-8 as `bytea`, the shape the rotation script's +-- `rotateBytesColumn` already covers). The column is registered in +-- ENCRYPTED_COLUMNS + the rotation script in the same change. +-- +-- ADDITIVE ONLY: existing rows keep NULL summaries; the on-demand transient +-- summary route is unchanged. +ALTER TABLE "inbound_documents" + ADD COLUMN IF NOT EXISTS "summary_encrypted" BYTEA, + ADD COLUMN IF NOT EXISTS "summary_generated_at" TIMESTAMP(3); diff --git a/prisma/schema.prisma b/prisma/schema.prisma index e71bca08a..b328fb331 100644 --- a/prisma/schema.prisma +++ b/prisma/schema.prisma @@ -6061,9 +6061,9 @@ enum ExtractedFactStatus { } model InboundDocument { - id String @id @default(cuid()) - userId String @map("user_id") - kind InboundDocumentKind @default(OTHER) + id String @id @default(cuid()) + userId String @map("user_id") + kind InboundDocumentKind @default(OTHER) /// User-given label. Deliberately plaintext (like `filename`) so the library /// list can ILIKE-search + ORDER BY in SQL. The document BODY stays /// encrypted; only this short, user-authored title is clear-text. Tradeoff: @@ -6071,18 +6071,18 @@ model InboundDocument { /// cleartext at rest / in backups — the same exposure the pre-existing /// plaintext `filename` already carries. Accepted to keep server-side search /// and sort. - title String? + title String? /// Original filename as uploaded (sanitised, optional). - filename String? + filename String? /// Magic-byte-sniffed MIME type the document was stored as. - mimeType String @map("mime_type") + mimeType String @map("mime_type") /// Stored byte length of the original (pre-encryption) document. - byteSize Int @map("byte_size") + byteSize Int @map("byte_size") /// AES-256-GCM ciphertext of the raw document bytes (base64 of the binary /// → `encrypt()` string → UTF-8 BYTEA), the `*_encrypted` Bytes convention /// shared with the note columns. The original is sensitive and never /// stored in the clear or logged. - contentEncrypted Bytes @map("content_encrypted") + contentEncrypted Bytes @map("content_encrypted") /// Lowercase-hex SHA-256 of the PLAINTEXT document bytes, computed at /// upload for same-user duplicate detection. Nullable: rows stored before /// the vault release are not backfilled (only new uploads dedupe). A @@ -6091,40 +6091,51 @@ model InboundDocument { /// CREATE UNIQUE INDEX inbound_documents_user_sha_live /// ON inbound_documents (user_id, content_sha256) /// WHERE deleted_at IS NULL AND content_sha256 IS NOT NULL; - contentSha256 String? @map("content_sha256") @db.VarChar(64) + contentSha256 String? @map("content_sha256") @db.VarChar(64) /// Storage codec of `contentEncrypted`. "base64v1" = legacy string path /// (base64-of-binary → `encrypt()` string → UTF-8 bytes); "binary2" = the /// binary AES-256-GCM layout written by `encryptBytes()` (no base64 /// detour). Explicit column, never header-sniffed; the default keeps every /// pre-existing row correct. - contentCodec String @default("base64v1") @map("content_codec") - status InboundDocumentStatus @default(STORED) + contentCodec String @default("base64v1") @map("content_codec") + status InboundDocumentStatus @default(STORED) /// Provider type that ran the extraction (e.g. "anthropic", "openai", /// "local"). Diagnostic only. Null on a store-only document. - providerType String? @map("provider_type") + providerType String? @map("provider_type") /// The document's stated report/collection date (YYYY-MM-DD at UTC), as /// transcribed by the model — never inferred, never user-set. - reportDate DateTime? @map("report_date") + reportDate DateTime? @map("report_date") /// Effective filing date the library sorts, groups, and range-filters on /// (editable). Distinct from `reportDate`, which is the model-transcribed /// value. On store it defaults to the upload day (now) when the user does /// not supply one, so a stored row's filing date is never NULL and display /// == sort == filter. It stays user-editable and MAY be cleared back to /// NULL via PATCH (the list then sorts such rows last). - documentDate DateTime? @map("document_date") + documentDate DateTime? @map("document_date") /// Failure reason when status = FAILED (a short code, never the document). - errorReason String? @map("error_reason") + errorReason String? @map("error_reason") + /// AES-256-GCM ciphertext of a short (3-4 sentence) plain-language summary + /// of WHAT the document is, generated ONCE in the background right after + /// upload when the user's `documentsAutoAiRead` opt-in is ON (that flag is + /// also the standing AI-egress consent). Absent when auto-read is OFF, no + /// provider is configured, or generation has not run yet. Descriptive only — + /// never a diagnosis. `encrypt()`-string-as-UTF-8 BYTEA, the shared Bytes + /// codec every other note column uses; PHI, never logged. + summaryEncrypted Bytes? @map("summary_encrypted") + /// When the background summary was generated (null until it runs). Surfaced + /// as meta beside the summary in the document detail view. + summaryGeneratedAt DateTime? @map("summary_generated_at") - facts ExtractedFact[] - conditionLinks DocumentConditionLink[] + facts ExtractedFact[] + conditionLinks DocumentConditionLink[] /// 1:1 blind content-search index (encrypted extracted text + HMAC token /// array). Absent until the document is indexed; cascades on delete. - contentIndex DocumentContentIndex? + contentIndex DocumentContentIndex? /// 1:1 encrypted preview thumbnail (small JPEG, AES-256-GCM at rest). /// Absent until the background thumbnail job runs; a side table (not a /// column) so the list/detail `omit: { contentEncrypted }` queries never /// drag the blob into the SELECT. Cascades on delete. - thumbnail DocumentThumbnail? + thumbnail DocumentThumbnail? /// v1.29.x (S7) — coach-conversation attachments (m:n via /// `CoachConversationDocument`). Deleting this document cascades its join rows /// away (its text leaves every conversation's context) but never the @@ -6132,7 +6143,7 @@ model InboundDocument { chatAttachments CoachConversationDocument[] /// share-link memberships (m:n via `ClinicianShareLinkDocument`). /// Cascades: deleting this document removes it from every share it was on. - shareLinks ClinicianShareLinkDocument[] + shareLinks ClinicianShareLinkDocument[] createdAt DateTime @default(now()) @map("created_at") updatedAt DateTime @updatedAt @map("updated_at") diff --git a/scripts/rotate-encryption-key.ts b/scripts/rotate-encryption-key.ts index 83b1df086..7ebf9f0dd 100644 --- a/scripts/rotate-encryption-key.ts +++ b/scripts/rotate-encryption-key.ts @@ -540,6 +540,18 @@ async function main() { prisma.extractedFact, ), ); + // The short background document summary. Small `encrypt()`-string-as-UTF-8 + // Bytes payload (never the multi-megabyte original blob — the select is + // scoped to id + this column), so the generic Bytes walk covers it. NULL on + // rows uploaded with auto-read OFF / no provider — `rotateBytesColumn` skips + // those. + results.push( + await rotateBytesColumn( + "InboundDocument", + "summaryEncrypted", + prisma.inboundDocument, + ), + ); // ───── v1.27.22 document content-search index (Bytes text + re-tokenise) ───── // The blind content index carries TWO coupled artefacts under the index key diff --git a/src/app/api/documents/inbound/route.ts b/src/app/api/documents/inbound/route.ts index ec2a6e313..f4a885e3c 100644 --- a/src/app/api/documents/inbound/route.ts +++ b/src/app/api/documents/inbound/route.ts @@ -42,6 +42,7 @@ import { type SerialisableDocument, } from "@/lib/documents/store"; import { enqueueDocumentIndex } from "@/lib/jobs/document-index"; +import { enqueueDocumentSummary } from "@/lib/jobs/document-summary"; import { enqueueDocumentThumbnail } from "@/lib/jobs/document-thumbnail"; import { detectDocumentType, @@ -378,6 +379,13 @@ async function postUpload(request: Request): Promise { // Only fresh inserts reach here (a duplicate returns early above). void enqueueDocumentThumbnail(user.id, document.id); + // Summarise the freshly stored document in the background — but ONLY when the + // `documentsAutoAiRead` opt-in is ON (the job re-checks it and the egress + // consent, and no-ops otherwise). The persisted summary shows on the detail + // view. Same fire-and-forget contract: the upload never blocks on or fails + // because of it. Only fresh inserts reach here (a duplicate returns early). + void enqueueDocumentSummary(user.id, document.id); + const links = await loadConditionLinks(user.id, [document.id]); return apiSuccess( serialiseDocument( diff --git a/src/components/documents/document-detail-sheet.tsx b/src/components/documents/document-detail-sheet.tsx index 4b0515e58..5c48c97de 100644 --- a/src/components/documents/document-detail-sheet.tsx +++ b/src/components/documents/document-detail-sheet.tsx @@ -570,6 +570,35 @@ export function DocumentDetailSheet({ )} + {doc.summary ? ( + // Persisted background summary — generated once at upload when the + // auto-read opt-in is on. AI narrative body → foreground content; + // the generation date is muted meta (UI-STANDARDS §3). +
+

+ {t("documents.detail.summary.title")} +

+

{doc.summary}

+ {doc.summaryGeneratedAt ? ( +

+ {format.date(doc.summaryGeneratedAt)} +

+ ) : null} +
+ ) : autoReadEnabled ? ( + // Auto-read is on but no summary yet (still generating, or the + // provider could not read this file) — one calm muted hint. +

+ {t("documents.detail.summary.pending")} +

+ ) : null} + { + return encryptToBytes(summary); +} + +/** Decrypt a stored document summary's `Bytes` payload. Throws on a bad key. */ +export function decryptDocumentSummary(buf: Uint8Array): string { + return decryptFromBytes(buf); +} + /** * Encrypt a staged fact's FHIR-staged payload into the `Bytes` column the * schema stores. The structured clinical values (diagnosis text, lab values, @@ -217,6 +236,18 @@ export function serialiseDocumentDetail( // `factCount` excludes REJECTED facts (a rejected fact is discarded, not part // of the document's tally) so the badge matches the list query. const factCount = facts.filter((f) => f.status !== "REJECTED").length; + // Decrypt the persisted background summary for the detail view. A decrypt + // failure (a rotated-away legacy key) degrades to no summary rather than + // failing the whole detail load — the field is descriptive, not load-bearing, + // and the ciphertext is never returned (fail-closed to null). + let summary: string | null = null; + if (doc.summaryEncrypted && doc.summaryEncrypted.byteLength > 0) { + try { + summary = decryptDocumentSummary(doc.summaryEncrypted); + } catch { + summary = null; + } + } return { ...serialiseDocument( doc, @@ -227,5 +258,9 @@ export function serialiseDocumentDetail( hasThumbnail, ), facts: facts.map(serialiseFact), + summary, + summaryGeneratedAt: doc.summaryGeneratedAt + ? doc.summaryGeneratedAt.toISOString() + : null, }; } diff --git a/src/lib/jobs/__tests__/document-summary.test.ts b/src/lib/jobs/__tests__/document-summary.test.ts new file mode 100644 index 000000000..1eded8223 --- /dev/null +++ b/src/lib/jobs/__tests__/document-summary.test.ts @@ -0,0 +1,191 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; + +/** + * The auto-summary background job's gating. Pins: the opt-in (`documentsAutoAiRead`) + * is the trigger — OFF → strictly no summary (no provider call, no write); ON + + * a configured provider → the summary is generated once and persisted ENCRYPTED + * (write scoped to `summaryEncrypted: null` so a re-run is a no-op); an already- + * summarised document short-circuits before the opt-in is even read; no provider + * configured → a graceful no-op; a provider error refunds the budget and persists + * nothing. + */ + +vi.mock("@/lib/logging/context", () => ({ annotate: vi.fn() })); +vi.mock("@/lib/db", () => ({ + prisma: { + inboundDocument: { + findFirst: vi.fn(), + updateMany: vi.fn(), + }, + }, +})); +vi.mock("@/lib/jobs/boss-instance", () => ({ getGlobalBoss: vi.fn() })); +vi.mock("@/lib/documents/document-settings", () => ({ + documentAutoReadEnabled: vi.fn(), +})); +vi.mock("@/lib/documents/provider-order", () => ({ + resolveDocumentVisionProvider: vi.fn(), +})); +vi.mock("@/lib/documents/ai-route-support", () => ({ + loadOwnedDocument: vi.fn(), + prepareVisionInput: vi.fn(), +})); +vi.mock("@/lib/documents/describe", () => ({ + runDocumentSummary: vi.fn(), +})); +vi.mock("@/lib/documents/store", () => ({ + encryptDocumentSummary: vi.fn(() => new Uint8Array([9, 9, 9])), +})); +vi.mock("@/lib/ai/consent-guard", () => { + class ConsentRequiredError extends Error {} + return { + ConsentRequiredError, + assertDocumentEgressConsent: vi.fn(), + }; +}); +vi.mock("@/lib/ai/coach/budget", () => ({ + buildDateKey: vi.fn(() => "2026-07-17"), + reserveBudget: vi.fn(), + reconcileSpend: vi.fn().mockResolvedValue(undefined), + resolveDailyCap: vi.fn(() => 1000), +})); +vi.mock("@/lib/ai/ai-budgets", () => ({ + AI_BUDGETS: { documentSummary: { temperature: 0.3, maxTokens: 600 } }, +})); + +import { runDocumentSummaryJob } from "../document-summary"; +import { prisma } from "@/lib/db"; +import { documentAutoReadEnabled } from "@/lib/documents/document-settings"; +import { resolveDocumentVisionProvider } from "@/lib/documents/provider-order"; +import { + loadOwnedDocument, + prepareVisionInput, +} from "@/lib/documents/ai-route-support"; +import { runDocumentSummary } from "@/lib/documents/describe"; +import { encryptDocumentSummary } from "@/lib/documents/store"; +import { assertDocumentEgressConsent } from "@/lib/ai/consent-guard"; +import { reserveBudget, reconcileSpend } from "@/lib/ai/coach/budget"; + +const DOC = { + id: "doc-1", + kind: "OTHER", + contentEncrypted: new Uint8Array([1, 2, 3]), + contentCodec: "binary2", + mimeType: "application/pdf", + status: "STORED", +}; + +/** An EXTERNAL (Anthropic) document pick — egress, so opt-in-gated. */ +const PICK = { + chain: [{ providerType: "anthropic", instance: {} }], + pick: { + entry: { providerType: "anthropic", instance: {} }, + providerType: "anthropic", + pdfSupported: true, + }, +}; + +function visionOk() { + vi.mocked(prepareVisionInput).mockResolvedValue({ + ok: true, + images: [], + documents: [{ mediaType: "application/pdf", dataBase64: "AA==" }], + } as never); +} + +beforeEach(() => { + vi.clearAllMocks(); + // Default: a live, not-yet-summarised document. + vi.mocked(prisma.inboundDocument.findFirst).mockResolvedValue({ + id: "doc-1", + summaryEncrypted: null, + } as never); + vi.mocked(prisma.inboundDocument.updateMany).mockResolvedValue({ + count: 1, + } as never); + vi.mocked(documentAutoReadEnabled).mockResolvedValue(true); + vi.mocked(resolveDocumentVisionProvider).mockResolvedValue(PICK as never); + vi.mocked(assertDocumentEgressConsent).mockResolvedValue(undefined); + vi.mocked(loadOwnedDocument).mockResolvedValue(DOC as never); + visionOk(); + vi.mocked(runDocumentSummary).mockResolvedValue({ + summary: "A lab report from a clinic listing routine blood values.", + } as never); + vi.mocked(reserveBudget).mockResolvedValue({ + allowed: true, + reserved: 5, + } as never); +}); + +describe("runDocumentSummaryJob — gating", () => { + it("does NOTHING when the opt-in is OFF (no provider call, no write)", async () => { + vi.mocked(documentAutoReadEnabled).mockResolvedValue(false); + + await runDocumentSummaryJob({ userId: "user-1", documentId: "doc-1" }); + + expect(resolveDocumentVisionProvider).not.toHaveBeenCalled(); + expect(runDocumentSummary).not.toHaveBeenCalled(); + expect(prisma.inboundDocument.updateMany).not.toHaveBeenCalled(); + }); + + it("generates and PERSISTS the summary encrypted when opt-in is ON + a provider is configured", async () => { + await runDocumentSummaryJob({ userId: "user-1", documentId: "doc-1" }); + + expect(runDocumentSummary).toHaveBeenCalledTimes(1); + expect(encryptDocumentSummary).toHaveBeenCalledWith( + "A lab report from a clinic listing routine blood values.", + ); + // Budget reserved then reconciled at the full reserved amount (charged). + expect(reconcileSpend).toHaveBeenCalledWith("user-1", 5, 5, "2026-07-17"); + // Write is scoped to the still-null column so a re-run cannot clobber it. + expect(prisma.inboundDocument.updateMany).toHaveBeenCalledWith( + expect.objectContaining({ + where: expect.objectContaining({ + id: "doc-1", + userId: "user-1", + summaryEncrypted: null, + }), + data: expect.objectContaining({ + summaryEncrypted: expect.any(Uint8Array), + summaryGeneratedAt: expect.any(Date), + }), + }), + ); + }); + + it("SKIPS a document that already has a summary (before reading the opt-in)", async () => { + vi.mocked(prisma.inboundDocument.findFirst).mockResolvedValue({ + id: "doc-1", + summaryEncrypted: new Uint8Array([1]), + } as never); + + await runDocumentSummaryJob({ userId: "user-1", documentId: "doc-1" }); + + expect(documentAutoReadEnabled).not.toHaveBeenCalled(); + expect(runDocumentSummary).not.toHaveBeenCalled(); + expect(prisma.inboundDocument.updateMany).not.toHaveBeenCalled(); + }); + + it("is a graceful no-op when NO provider is configured", async () => { + vi.mocked(resolveDocumentVisionProvider).mockResolvedValue({ + chain: [], + pick: null, + } as never); + + await runDocumentSummaryJob({ userId: "user-1", documentId: "doc-1" }); + + expect(reserveBudget).not.toHaveBeenCalled(); + expect(runDocumentSummary).not.toHaveBeenCalled(); + expect(prisma.inboundDocument.updateMany).not.toHaveBeenCalled(); + }); + + it("refunds the budget and persists nothing when the provider call throws", async () => { + vi.mocked(runDocumentSummary).mockRejectedValueOnce(new Error("boom")); + + await runDocumentSummaryJob({ userId: "user-1", documentId: "doc-1" }); + + // Reservation refunded to zero spend; no write. + expect(reconcileSpend).toHaveBeenCalledWith("user-1", 5, 0, "2026-07-17"); + expect(prisma.inboundDocument.updateMany).not.toHaveBeenCalled(); + }); +}); diff --git a/src/lib/jobs/document-summary.ts b/src/lib/jobs/document-summary.ts new file mode 100644 index 000000000..b5134ea00 --- /dev/null +++ b/src/lib/jobs/document-summary.ts @@ -0,0 +1,243 @@ +/** + * Automatic per-document plain-language summary, enqueued the moment a document + * is stored (`POST /api/documents/inbound`). Fire-and-forget: the upload + * response never blocks on or fails because of it. + * + * Unlike the on-demand summary route (which is transient — nothing persisted), + * THIS path persists a short (3-4 sentence) descriptive summary ENCRYPTED on the + * document row, generated ONCE. It runs ONLY when the user's `documentsAutoAiRead` + * opt-in is ON — that flag is the standing AI-egress consent that removes the + * per-document friction. With the opt-in OFF the job is a strict no-op and the + * on-demand route stays the only way to get a summary. + * + * The gate order mirrors the vault's other AI paths (module gate is enforced at + * the enqueueing upload route): load → opt-in ON → provider resolved (DOCUMENT + * order, local-first) → egress consent re-asserted → budget reserved → generate + * → persist encrypted. Every skip is graceful (no throw, no retry): an already- + * summarised document, no configured provider, a spent budget, or a document the + * picked provider cannot read all resolve to a clean no-op. The summary is + * DESCRIPTIVE ONLY (interpretation boundary G7, enforced by `runDocumentSummary`). + * + * The queue MUST be registered in the maintenance registrar + * (`src/lib/jobs/reminder/register-maintenance.ts`) so pg-boss provisions it. + */ +import { AI_BUDGETS } from "@/lib/ai/ai-budgets"; +import { + assertDocumentEgressConsent, + ConsentRequiredError, +} from "@/lib/ai/consent-guard"; +import { + buildDateKey, + reconcileSpend, + reserveBudget, + resolveDailyCap, +} from "@/lib/ai/coach/budget"; +import { + loadOwnedDocument, + prepareVisionInput, +} from "@/lib/documents/ai-route-support"; +import { runDocumentSummary } from "@/lib/documents/describe"; +import { documentAutoReadEnabled } from "@/lib/documents/document-settings"; +import { resolveDocumentVisionProvider } from "@/lib/documents/provider-order"; +import { encryptDocumentSummary } from "@/lib/documents/store"; +import { prisma } from "@/lib/db"; +import { getGlobalBoss } from "@/lib/jobs/boss-instance"; +import { annotate } from "@/lib/logging/context"; + +export const DOCUMENT_SUMMARY_QUEUE = "document-summary"; + +/** Serial concurrency — provider calls, kept off the request pool. */ +export const DOCUMENT_SUMMARY_CONCURRENCY = 1; + +export interface DocumentSummaryPayload { + userId: string; + documentId: string; + enqueuedAt?: string; +} + +/** + * Generate + persist ONE document's background summary (owner-scoped). Never + * throws for an expected outcome — every gate that fails resolves to a tagged + * no-op so pg-boss does not retry a document that simply has no provider or a + * user who opted out. Idempotent: a document that already carries a summary is + * skipped, and the final write only lands when the column is still null. + */ +export async function runDocumentSummaryJob( + payload: DocumentSummaryPayload, +): Promise { + const { userId, documentId } = payload; + if (!userId || !documentId) return; + + // Idempotency: a document that already has a summary is a cheap no-op (a + // re-enqueue after a duplicate upload, or a boot backfill overlap). + const existing = await prisma.inboundDocument.findFirst({ + where: { id: documentId, userId, deletedAt: null }, + select: { id: true, summaryEncrypted: true }, + }); + if (!existing || existing.summaryEncrypted) { + annotate({ + action: { name: "documents.summary.autoSkipped" }, + meta: { documentId, reason: !existing ? "not-found" : "exists" }, + }); + return; + } + + // The opt-in IS the trigger: when it is OFF the auto path does nothing (the + // on-demand route stays the only way to summarise). Fail-closed on a missing + // row. + if (!(await documentAutoReadEnabled(userId))) { + annotate({ + action: { name: "documents.summary.autoSkipped" }, + meta: { documentId, reason: "opt-out" }, + }); + return; + } + + // Resolve the DOCUMENT-order vision provider (local-first, codex last). No + // vision-capable provider configured → graceful no-op (unlike the index job, + // there is no local text-layer fallback for a descriptive summary). + const { pick } = await resolveDocumentVisionProvider(userId); + if (!pick) { + annotate({ + action: { name: "documents.summary.autoSkipped" }, + meta: { documentId, reason: "no-provider" }, + }); + return; + } + + // Re-assert egress consent for the picked provider. A local pick is ungated; + // an external pick is authorised by the `documentsAutoAiRead` opt-in checked + // above (the toggle short-circuits the gate). Belt-and-braces: a consent race + // (opt-out flipped mid-flight) resolves to a no-op, never an egress. + try { + await assertDocumentEgressConsent({ + userId, + providerType: pick.providerType, + surface: "insights", + }); + } catch (err) { + annotate({ + action: { name: "documents.summary.autoSkipped" }, + meta: { + documentId, + reason: + err instanceof ConsentRequiredError ? "consent" : "consent-error", + }, + }); + return; + } + + const document = await loadOwnedDocument(userId, documentId); + if (!document) return; + + const vision = await prepareVisionInput(document, pick.pdfSupported); + if (!vision.ok) { + // Cannot read this file with the picked provider (decrypt miss, unsupported + // type, or a PDF needing an Anthropic provider). Graceful no-op. + annotate({ + action: { name: "documents.summary.autoSkipped" }, + meta: { documentId, reason: vision.reason }, + }); + return; + } + + const dateKey = buildDateKey(); + const reservation = await reserveBudget( + userId, + AI_BUDGETS.documentSummary.maxTokens, + dateKey, + resolveDailyCap([{ providerType: pick.entry.providerType }]), + ); + // Budget exhausted → skip (no local fallback for a summary). + if (!reservation.allowed) { + annotate({ + action: { name: "documents.summary.autoSkipped" }, + meta: { documentId, reason: "budget" }, + }); + return; + } + + try { + const { summary } = await runDocumentSummary({ + provider: pick.entry.instance, + providerType: pick.providerType, + images: vision.images, + documents: vision.documents, + }); + await reconcileSpend( + userId, + reservation.reserved, + reservation.reserved, + dateKey, + ); + // Persist ENCRYPTED. `updateMany` scoped to `summaryEncrypted: null` so a + // racing writer (a re-enqueue) never overwrites an existing summary — the + // first write wins and a second run is a no-op. + const written = await prisma.inboundDocument.updateMany({ + where: { + id: documentId, + userId, + deletedAt: null, + summaryEncrypted: null, + }, + data: { + summaryEncrypted: encryptDocumentSummary(summary), + summaryGeneratedAt: new Date(), + }, + }); + annotate({ + action: { name: "documents.summary.autoGenerated" }, + meta: { + documentId, + providerType: pick.providerType, + length: summary.length, + persisted: written.count > 0, + }, + }); + } catch { + // Refund the reservation on a provider miss; the document keeps no summary + // (the on-demand route remains the manual fallback). Never rethrow — a + // transient provider error must not retry-loop or fail the queue. + await reconcileSpend(userId, reservation.reserved, 0, dateKey); + annotate({ + action: { name: "documents.summary.autoFailed" }, + meta: { documentId, reason: "provider_error" }, + }); + } +} + +/** + * Enqueue a per-document summary job. Coalesced by `singletonKey` per document + * so a duplicate/idempotent re-upload cannot pile up parallel runs. Fire-and- + * forget — a missing boss (worker not up) is a no-op, and a `boss.send` failure + * (a transient DB hiccup) is swallowed to a no-op too, so enqueue can NEVER fail + * an already-stored upload. The summary is a background nicety; the stored + * document is the contract. + */ +export async function enqueueDocumentSummary( + userId: string, + documentId: string, +): Promise<{ enqueued: boolean }> { + const boss = getGlobalBoss(); + if (!boss) return { enqueued: false }; + const payload: DocumentSummaryPayload = { + userId, + documentId, + enqueuedAt: new Date().toISOString(), + }; + try { + const jobId = await boss.send(DOCUMENT_SUMMARY_QUEUE, payload, { + retryLimit: 2, + retryDelay: 30, + retryBackoff: true, + singletonKey: `document-summary|${documentId}`, + }); + return { enqueued: Boolean(jobId) }; + } catch (err) { + annotate({ + action: { name: "documents.summary.enqueueFailed" }, + meta: { documentId, reason: err instanceof Error ? err.name : "unknown" }, + }); + return { enqueued: false }; + } +} diff --git a/src/lib/jobs/reminder/register-maintenance.ts b/src/lib/jobs/reminder/register-maintenance.ts index 56f27bae3..62053f2c2 100644 --- a/src/lib/jobs/reminder/register-maintenance.ts +++ b/src/lib/jobs/reminder/register-maintenance.ts @@ -87,6 +87,12 @@ import { runDocumentThumbnail, type DocumentThumbnailPayload, } from "@/lib/jobs/document-thumbnail"; +import { + DOCUMENT_SUMMARY_QUEUE, + DOCUMENT_SUMMARY_CONCURRENCY, + runDocumentSummaryJob, + type DocumentSummaryPayload, +} from "@/lib/jobs/document-summary"; import { DOCUMENT_THUMBNAIL_BACKFILL_QUEUE, DOCUMENT_THUMBNAIL_BACKFILL_CONCURRENCY, @@ -344,6 +350,11 @@ const allQueues = [ // thumbnail jobs. Without this entry pg-boss never provisions the queue and // the boot discovery silently no-ops. DOCUMENT_THUMBNAIL_BACKFILL_QUEUE, + // Document AI — automatic per-document plain-language summary, enqueued on + // upload. Provider (vision) only, gated on the `documentsAutoAiRead` opt-in + + // egress consent + budget; persists the summary encrypted. Without this entry + // pg-boss never provisions the queue and every upload enqueue silently no-ops. + DOCUMENT_SUMMARY_QUEUE, ]; const schedules: ScheduleEntry[] = [ @@ -768,6 +779,33 @@ export async function registerMaintenanceQueues( }, ); + // Document AI — automatic per-document plain-language summary. Enqueued on + // upload (one job per stored document); runs the vision provider ONLY when + // the `documentsAutoAiRead` opt-in is ON (egress consent + budget gated) and + // persists the summary encrypted. Serial concurrency so the provider call + // never crowds the request pool. + await boss.work( + DOCUMENT_SUMMARY_QUEUE, + { localConcurrency: DOCUMENT_SUMMARY_CONCURRENCY }, + async (jobs) => { + for (const job of jobs) { + const { userId, documentId } = job.data; + if (!userId || !documentId) continue; + try { + await runDocumentSummaryJob(job.data); + } catch (err) { + recordError(); + workerLog( + "error", + `[document-summary] user=${userId} document=${documentId} failed`, + err, + ); + throw err; + } + } + }, + ); + // Document vault — boot-time preview-thumbnail backfill worker. The boot // discovery sends one per-user job; this handler fans out per-document // thumbnail jobs for that user's not-yet-thumbnailed documents. Serial diff --git a/src/lib/openapi/routes/documents.ts b/src/lib/openapi/routes/documents.ts index b09bc02d1..2915f0170 100644 --- a/src/lib/openapi/routes/documents.ts +++ b/src/lib/openapi/routes/documents.ts @@ -161,8 +161,16 @@ const inboundDocument = z }); const inboundDocumentDetail = inboundDocument - .extend({ facts: z.array(extractedFact) }) - .meta({ id: "InboundDocumentDetail" }); + .extend({ + facts: z.array(extractedFact), + summary: z.string().nullable(), + summaryGeneratedAt: z.string().nullable(), + }) + .meta({ + id: "InboundDocumentDetail", + description: + "A stored document plus its staged facts. `summary` is a short (3-4 sentence) plain-language description of WHAT the document is, generated once in the background right after upload when the `documentsAutoAiRead` opt-in is ON (that flag is also the standing AI-egress consent); it is descriptive only, never a diagnosis, and is null when auto-read is OFF, no provider is configured, or the background summary has not run yet. `summaryGeneratedAt` is when it was generated (null until then).", + }); const listResponse = z .object({ diff --git a/src/lib/validations/inbound-documents.ts b/src/lib/validations/inbound-documents.ts index dcda57f3d..0b7d381da 100644 --- a/src/lib/validations/inbound-documents.ts +++ b/src/lib/validations/inbound-documents.ts @@ -292,6 +292,15 @@ export interface InboundDocumentDto { export interface InboundDocumentDetailDto extends InboundDocumentDto { facts: ExtractedFactDto[]; + /** + * Short (3-4 sentence) plain-language summary of WHAT the document is, + * generated once in the background after upload when the `documentsAutoAiRead` + * opt-in is ON. Null when auto-read is OFF, no provider is configured, or the + * background summary has not run yet. Descriptive only — never a diagnosis. + */ + summary: string | null; + /** When the background summary was generated (ISO 8601), or null. */ + summaryGeneratedAt: string | null; } // ─── Edit (correction before approval) ─────────────────────────────────────