Merge pull request #523 from MBombeck/feat/document-auto-summary

feat(documents): persist an auto summary for uploaded documents
This commit is contained in:
Marc-André Bombeck authored and GitHub committed 2026-07-17 22:41:30 +02:00
commit 0a08b166cf
19 files changed
+674 -22

No files matched your search

+15
View File
@@ -29085,6 +29085,14 @@ components:
type: array
items:
$ref: "#/components/schemas/ExtractedFact"
summary:
anyOf:
- type: string
- type: "null"
summaryGeneratedAt:
anyOf:
- type: string
- type: "null"
required:
- id
- kind
@@ -29107,7 +29115,14 @@ components:
- createdAt
- updatedAt
- facts
- summary
- summaryGeneratedAt
additionalProperties: false
description: A stored document plus its staged facts. `summary` is a short (3-4 sentence) plain-language description of
WHAT the document is, generated once in the background right after upload when the `documentsAutoAiRead` opt-in
is ON (that flag is also the standing AI-egress consent); it is descriptive only, never a diagnosis, and is null
when auto-read is OFF, no provider is configured, or the background summary has not run yet.
`summaryGeneratedAt` is when it was generated (null until then).
ExtractedFact:
type: object
properties:
+4
View File
@@ -8331,6 +8331,10 @@
},
"detail": {
"title": "Dokumentdetails",
"summary": {
"title": "Zusammenfassung",
"pending": "Die Zusammenfassung wird erstellt."
},
"loadError": "Dieses Dokument konnte nicht geladen werden.",
"saveError": "Die Änderung konnte nicht gespeichert werden.",
"deleteError": "Das Dokument konnte nicht gelöscht werden.",
+4
View File
@@ -8331,6 +8331,10 @@
},
"detail": {
"title": "Document details",
"summary": {
"title": "Summary",
"pending": "The summary is being prepared."
},
"loadError": "Couldn't load this document.",
"saveError": "Couldn't save the change.",
"deleteError": "Couldn't delete the document.",
+4
View File
@@ -8331,6 +8331,10 @@
},
"detail": {
"title": "Detalles del documento",
"summary": {
"title": "Resumen",
"pending": "Se está preparando el resumen."
},
"loadError": "No se pudo cargar este documento.",
"saveError": "No se pudo guardar el cambio.",
"deleteError": "No se pudo eliminar el documento.",
+4
View File
@@ -8331,6 +8331,10 @@
},
"detail": {
"title": "Détails du document",
"summary": {
"title": "Résumé",
"pending": "Le résumé est en cours de préparation."
},
"loadError": "Impossible de charger ce document.",
"saveError": "Impossible d'enregistrer la modification.",
"deleteError": "Impossible de supprimer le document.",
+4
View File
@@ -8331,6 +8331,10 @@
},
"detail": {
"title": "Dettagli del documento",
"summary": {
"title": "Riepilogo",
"pending": "Il riepilogo è in preparazione."
},
"loadError": "Impossibile caricare questo documento.",
"saveError": "Impossibile salvare la modifica.",
"deleteError": "Impossibile eliminare il documento.",
+4
View File
@@ -8331,6 +8331,10 @@
},
"detail": {
"title": "Szczegóły dokumentu",
"summary": {
"title": "Podsumowanie",
"pending": "Podsumowanie jest przygotowywane."
},
"loadError": "Nie udało się wczytać tego dokumentu.",
"saveError": "Nie udało się zapisać zmiany.",
"deleteError": "Nie udało się usunąć dokumentu.",
@@ -0,0 +1,23 @@
-- Persist a short plain-language summary of an uploaded document.
--
-- Adds two nullable columns to `inbound_documents`:
-- - `summary_encrypted` — AES-256-GCM ciphertext of a 3-4 sentence
-- plain-language description of WHAT the document is, generated ONCE in
-- the background right after upload when the user's `documents_auto_ai_read`
-- opt-in is ON (that flag is also the standing AI-egress consent). NULL
-- when auto-read is OFF, no provider is configured, or generation has not
-- run yet.
-- - `summary_generated_at` — when that background summary was generated
-- (NULL until it runs).
--
-- STORAGE CONVENTION: Bytes (`bytea`), matching the dominant free-text
-- encrypted-note convention already in the schema (the `encrypt()` ciphertext
-- string stored UTF-8 as `bytea`, the shape the rotation script's
-- `rotateBytesColumn` already covers). The column is registered in
-- ENCRYPTED_COLUMNS + the rotation script in the same change.
--
-- ADDITIVE ONLY: existing rows keep NULL summaries; the on-demand transient
-- summary route is unchanged.
ALTER TABLE "inbound_documents"
ADD COLUMN IF NOT EXISTS "summary_encrypted" BYTEA,
ADD COLUMN IF NOT EXISTS "summary_generated_at" TIMESTAMP(3);
+31 -20
View File
@@ -6061,9 +6061,9 @@ enum ExtractedFactStatus {
}
model InboundDocument {
id String @id @default(cuid())
userId String @map("user_id")
kind InboundDocumentKind @default(OTHER)
id String @id @default(cuid())
userId String @map("user_id")
kind InboundDocumentKind @default(OTHER)
/// User-given label. Deliberately plaintext (like `filename`) so the library
/// list can ILIKE-search + ORDER BY in SQL. The document BODY stays
/// encrypted; only this short, user-authored title is clear-text. Tradeoff:
@@ -6071,18 +6071,18 @@ model InboundDocument {
/// cleartext at rest / in backups — the same exposure the pre-existing
/// plaintext `filename` already carries. Accepted to keep server-side search
/// and sort.
title String?
title String?
/// Original filename as uploaded (sanitised, optional).
filename String?
filename String?
/// Magic-byte-sniffed MIME type the document was stored as.
mimeType String @map("mime_type")
mimeType String @map("mime_type")
/// Stored byte length of the original (pre-encryption) document.
byteSize Int @map("byte_size")
byteSize Int @map("byte_size")
/// AES-256-GCM ciphertext of the raw document bytes (base64 of the binary
/// → `encrypt()` string → UTF-8 BYTEA), the `*_encrypted` Bytes convention
/// shared with the note columns. The original is sensitive and never
/// stored in the clear or logged.
contentEncrypted Bytes @map("content_encrypted")
contentEncrypted Bytes @map("content_encrypted")
/// Lowercase-hex SHA-256 of the PLAINTEXT document bytes, computed at
/// upload for same-user duplicate detection. Nullable: rows stored before
/// the vault release are not backfilled (only new uploads dedupe). A
@@ -6091,40 +6091,51 @@ model InboundDocument {
/// CREATE UNIQUE INDEX inbound_documents_user_sha_live
/// ON inbound_documents (user_id, content_sha256)
/// WHERE deleted_at IS NULL AND content_sha256 IS NOT NULL;
contentSha256 String? @map("content_sha256") @db.VarChar(64)
contentSha256 String? @map("content_sha256") @db.VarChar(64)
/// Storage codec of `contentEncrypted`. "base64v1" = legacy string path
/// (base64-of-binary → `encrypt()` string → UTF-8 bytes); "binary2" = the
/// binary AES-256-GCM layout written by `encryptBytes()` (no base64
/// detour). Explicit column, never header-sniffed; the default keeps every
/// pre-existing row correct.
contentCodec String @default("base64v1") @map("content_codec")
status InboundDocumentStatus @default(STORED)
contentCodec String @default("base64v1") @map("content_codec")
status InboundDocumentStatus @default(STORED)
/// Provider type that ran the extraction (e.g. "anthropic", "openai",
/// "local"). Diagnostic only. Null on a store-only document.
providerType String? @map("provider_type")
providerType String? @map("provider_type")
/// The document's stated report/collection date (YYYY-MM-DD at UTC), as
/// transcribed by the model — never inferred, never user-set.
reportDate DateTime? @map("report_date")
reportDate DateTime? @map("report_date")
/// Effective filing date the library sorts, groups, and range-filters on
/// (editable). Distinct from `reportDate`, which is the model-transcribed
/// value. On store it defaults to the upload day (now) when the user does
/// not supply one, so a stored row's filing date is never NULL and display
/// == sort == filter. It stays user-editable and MAY be cleared back to
/// NULL via PATCH (the list then sorts such rows last).
documentDate DateTime? @map("document_date")
documentDate DateTime? @map("document_date")
/// Failure reason when status = FAILED (a short code, never the document).
errorReason String? @map("error_reason")
errorReason String? @map("error_reason")
/// AES-256-GCM ciphertext of a short (3-4 sentence) plain-language summary
/// of WHAT the document is, generated ONCE in the background right after
/// upload when the user's `documentsAutoAiRead` opt-in is ON (that flag is
/// also the standing AI-egress consent). Absent when auto-read is OFF, no
/// provider is configured, or generation has not run yet. Descriptive only —
/// never a diagnosis. `encrypt()`-string-as-UTF-8 BYTEA, the shared Bytes
/// codec every other note column uses; PHI, never logged.
summaryEncrypted Bytes? @map("summary_encrypted")
/// When the background summary was generated (null until it runs). Surfaced
/// as meta beside the summary in the document detail view.
summaryGeneratedAt DateTime? @map("summary_generated_at")
facts ExtractedFact[]
conditionLinks DocumentConditionLink[]
facts ExtractedFact[]
conditionLinks DocumentConditionLink[]
/// 1:1 blind content-search index (encrypted extracted text + HMAC token
/// array). Absent until the document is indexed; cascades on delete.
contentIndex DocumentContentIndex?
contentIndex DocumentContentIndex?
/// 1:1 encrypted preview thumbnail (small JPEG, AES-256-GCM at rest).
/// Absent until the background thumbnail job runs; a side table (not a
/// column) so the list/detail `omit: { contentEncrypted }` queries never
/// drag the blob into the SELECT. Cascades on delete.
thumbnail DocumentThumbnail?
thumbnail DocumentThumbnail?
/// v1.29.x (S7) — coach-conversation attachments (m:n via
/// `CoachConversationDocument`). Deleting this document cascades its join rows
/// away (its text leaves every conversation's context) but never the
@@ -6132,7 +6143,7 @@ model InboundDocument {
chatAttachments CoachConversationDocument[]
/// share-link memberships (m:n via `ClinicianShareLinkDocument`).
/// Cascades: deleting this document removes it from every share it was on.
shareLinks ClinicianShareLinkDocument[]
shareLinks ClinicianShareLinkDocument[]
createdAt DateTime @default(now()) @map("created_at")
updatedAt DateTime @updatedAt @map("updated_at")
+12
View File
@@ -540,6 +540,18 @@ async function main() {
prisma.extractedFact,
),
);
// The short background document summary. Small `encrypt()`-string-as-UTF-8
// Bytes payload (never the multi-megabyte original blob — the select is
// scoped to id + this column), so the generic Bytes walk covers it. NULL on
// rows uploaded with auto-read OFF / no provider — `rotateBytesColumn` skips
// those.
results.push(
await rotateBytesColumn(
"InboundDocument",
"summaryEncrypted",
prisma.inboundDocument,
),
);
// ───── v1.27.22 document content-search index (Bytes text + re-tokenise) ─────
// The blind content index carries TWO coupled artefacts under the index key
+8
View File
@@ -42,6 +42,7 @@ import {
type SerialisableDocument,
} from "@/lib/documents/store";
import { enqueueDocumentIndex } from "@/lib/jobs/document-index";
import { enqueueDocumentSummary } from "@/lib/jobs/document-summary";
import { enqueueDocumentThumbnail } from "@/lib/jobs/document-thumbnail";
import {
detectDocumentType,
@@ -378,6 +379,13 @@ async function postUpload(request: Request): Promise<Response> {
// Only fresh inserts reach here (a duplicate returns early above).
void enqueueDocumentThumbnail(user.id, document.id);
// Summarise the freshly stored document in the background — but ONLY when the
// `documentsAutoAiRead` opt-in is ON (the job re-checks it and the egress
// consent, and no-ops otherwise). The persisted summary shows on the detail
// view. Same fire-and-forget contract: the upload never blocks on or fails
// because of it. Only fresh inserts reach here (a duplicate returns early).
void enqueueDocumentSummary(user.id, document.id);
const links = await loadConditionLinks(user.id, [document.id]);
return apiSuccess(
serialiseDocument(
@@ -570,6 +570,35 @@ export function DocumentDetailSheet({
</div>
)}
{doc.summary ? (
// Persisted background summary — generated once at upload when the
// auto-read opt-in is on. AI narrative body → foreground content;
// the generation date is muted meta (UI-STANDARDS §3).
<section
className="space-y-1.5"
data-slot="document-detail-summary"
>
<p className="text-sm leading-none font-medium">
{t("documents.detail.summary.title")}
</p>
<p className="text-foreground text-sm">{doc.summary}</p>
{doc.summaryGeneratedAt ? (
<p className="text-muted-foreground text-xs">
{format.date(doc.summaryGeneratedAt)}
</p>
) : null}
</section>
) : autoReadEnabled ? (
// Auto-read is on but no summary yet (still generating, or the
// provider could not read this file) — one calm muted hint.
<p
className="text-muted-foreground text-xs"
data-slot="document-detail-summary-pending"
>
{t("documents.detail.summary.pending")}
</p>
) : null}
<DocumentAiSection
aiEnabled={aiEnabled}
autoReadEnabled={autoReadEnabled}
+6
View File
@@ -229,6 +229,12 @@ export const ENCRYPTED_COLUMNS: readonly EncryptedColumn[] = [
// document itself rather than persisting as plaintext JSONB.
{ model: "ExtractedFact", field: "dataEncrypted", kind: "bytes" },
{ model: "ExtractedFact", field: "provenanceEncrypted", kind: "bytes" },
// The short plain-language document summary, generated once in the background
// after upload. Stored as the `encrypt()`-string-as-UTF-8 Bytes shape (the
// same codec the content index + note columns use), so the generic Bytes walk
// re-encrypts it. Nullable — pre-summary and opt-out rows are skipped
// generically, as for every other nullable Bytes column.
{ model: "InboundDocument", field: "summaryEncrypted", kind: "bytes" },
// ───── v1.27.22 document content-search index (Bytes text) ─────
// The normalised extracted text of an indexed document, AES-256-GCM at rest
+35
View File
@@ -113,6 +113,25 @@ export function decryptThumbnail(buf: Uint8Array): Buffer {
return Buffer.from(decryptFromBytes(buf), "base64");
}
/**
* Encrypt a document's short plain-language summary into the `Bytes` payload
* the schema stores (`InboundDocument.summaryEncrypted`). Uses the shared
* AES-256-GCM string codec (`encrypt()` string → UTF-8 bytes), the same shape
* `DocumentContentIndex.textEncrypted` uses — so the standard key-rotation walk
* (`rotateBytesColumn`) covers it. The summary is descriptive PHI; same posture
* (versioned key id, fail-closed decrypt) as every other encrypted column.
*/
export function encryptDocumentSummary(
summary: string,
): Uint8Array<ArrayBuffer> {
return encryptToBytes(summary);
}
/** Decrypt a stored document summary's `Bytes` payload. Throws on a bad key. */
export function decryptDocumentSummary(buf: Uint8Array): string {
return decryptFromBytes(buf);
}
/**
* Encrypt a staged fact's FHIR-staged payload into the `Bytes` column the
* schema stores. The structured clinical values (diagnosis text, lab values,
@@ -217,6 +236,18 @@ export function serialiseDocumentDetail(
// `factCount` excludes REJECTED facts (a rejected fact is discarded, not part
// of the document's tally) so the badge matches the list query.
const factCount = facts.filter((f) => f.status !== "REJECTED").length;
// Decrypt the persisted background summary for the detail view. A decrypt
// failure (a rotated-away legacy key) degrades to no summary rather than
// failing the whole detail load — the field is descriptive, not load-bearing,
// and the ciphertext is never returned (fail-closed to null).
let summary: string | null = null;
if (doc.summaryEncrypted && doc.summaryEncrypted.byteLength > 0) {
try {
summary = decryptDocumentSummary(doc.summaryEncrypted);
} catch {
summary = null;
}
}
return {
...serialiseDocument(
doc,
@@ -227,5 +258,9 @@ export function serialiseDocumentDetail(
hasThumbnail,
),
facts: facts.map(serialiseFact),
summary,
summaryGeneratedAt: doc.summaryGeneratedAt
? doc.summaryGeneratedAt.toISOString()
: null,
};
}
@@ -0,0 +1,191 @@
import { beforeEach, describe, expect, it, vi } from "vitest";
/**
* The auto-summary background job's gating. Pins: the opt-in (`documentsAutoAiRead`)
* is the trigger — OFF → strictly no summary (no provider call, no write); ON +
* a configured provider → the summary is generated once and persisted ENCRYPTED
* (write scoped to `summaryEncrypted: null` so a re-run is a no-op); an already-
* summarised document short-circuits before the opt-in is even read; no provider
* configured → a graceful no-op; a provider error refunds the budget and persists
* nothing.
*/
vi.mock("@/lib/logging/context", () => ({ annotate: vi.fn() }));
vi.mock("@/lib/db", () => ({
prisma: {
inboundDocument: {
findFirst: vi.fn(),
updateMany: vi.fn(),
},
},
}));
vi.mock("@/lib/jobs/boss-instance", () => ({ getGlobalBoss: vi.fn() }));
vi.mock("@/lib/documents/document-settings", () => ({
documentAutoReadEnabled: vi.fn(),
}));
vi.mock("@/lib/documents/provider-order", () => ({
resolveDocumentVisionProvider: vi.fn(),
}));
vi.mock("@/lib/documents/ai-route-support", () => ({
loadOwnedDocument: vi.fn(),
prepareVisionInput: vi.fn(),
}));
vi.mock("@/lib/documents/describe", () => ({
runDocumentSummary: vi.fn(),
}));
vi.mock("@/lib/documents/store", () => ({
encryptDocumentSummary: vi.fn(() => new Uint8Array([9, 9, 9])),
}));
vi.mock("@/lib/ai/consent-guard", () => {
class ConsentRequiredError extends Error {}
return {
ConsentRequiredError,
assertDocumentEgressConsent: vi.fn(),
};
});
vi.mock("@/lib/ai/coach/budget", () => ({
buildDateKey: vi.fn(() => "2026-07-17"),
reserveBudget: vi.fn(),
reconcileSpend: vi.fn().mockResolvedValue(undefined),
resolveDailyCap: vi.fn(() => 1000),
}));
vi.mock("@/lib/ai/ai-budgets", () => ({
AI_BUDGETS: { documentSummary: { temperature: 0.3, maxTokens: 600 } },
}));
import { runDocumentSummaryJob } from "../document-summary";
import { prisma } from "@/lib/db";
import { documentAutoReadEnabled } from "@/lib/documents/document-settings";
import { resolveDocumentVisionProvider } from "@/lib/documents/provider-order";
import {
loadOwnedDocument,
prepareVisionInput,
} from "@/lib/documents/ai-route-support";
import { runDocumentSummary } from "@/lib/documents/describe";
import { encryptDocumentSummary } from "@/lib/documents/store";
import { assertDocumentEgressConsent } from "@/lib/ai/consent-guard";
import { reserveBudget, reconcileSpend } from "@/lib/ai/coach/budget";
const DOC = {
id: "doc-1",
kind: "OTHER",
contentEncrypted: new Uint8Array([1, 2, 3]),
contentCodec: "binary2",
mimeType: "application/pdf",
status: "STORED",
};
/** An EXTERNAL (Anthropic) document pick — egress, so opt-in-gated. */
const PICK = {
chain: [{ providerType: "anthropic", instance: {} }],
pick: {
entry: { providerType: "anthropic", instance: {} },
providerType: "anthropic",
pdfSupported: true,
},
};
function visionOk() {
vi.mocked(prepareVisionInput).mockResolvedValue({
ok: true,
images: [],
documents: [{ mediaType: "application/pdf", dataBase64: "AA==" }],
} as never);
}
beforeEach(() => {
vi.clearAllMocks();
// Default: a live, not-yet-summarised document.
vi.mocked(prisma.inboundDocument.findFirst).mockResolvedValue({
id: "doc-1",
summaryEncrypted: null,
} as never);
vi.mocked(prisma.inboundDocument.updateMany).mockResolvedValue({
count: 1,
} as never);
vi.mocked(documentAutoReadEnabled).mockResolvedValue(true);
vi.mocked(resolveDocumentVisionProvider).mockResolvedValue(PICK as never);
vi.mocked(assertDocumentEgressConsent).mockResolvedValue(undefined);
vi.mocked(loadOwnedDocument).mockResolvedValue(DOC as never);
visionOk();
vi.mocked(runDocumentSummary).mockResolvedValue({
summary: "A lab report from a clinic listing routine blood values.",
} as never);
vi.mocked(reserveBudget).mockResolvedValue({
allowed: true,
reserved: 5,
} as never);
});
describe("runDocumentSummaryJob — gating", () => {
it("does NOTHING when the opt-in is OFF (no provider call, no write)", async () => {
vi.mocked(documentAutoReadEnabled).mockResolvedValue(false);
await runDocumentSummaryJob({ userId: "user-1", documentId: "doc-1" });
expect(resolveDocumentVisionProvider).not.toHaveBeenCalled();
expect(runDocumentSummary).not.toHaveBeenCalled();
expect(prisma.inboundDocument.updateMany).not.toHaveBeenCalled();
});
it("generates and PERSISTS the summary encrypted when opt-in is ON + a provider is configured", async () => {
await runDocumentSummaryJob({ userId: "user-1", documentId: "doc-1" });
expect(runDocumentSummary).toHaveBeenCalledTimes(1);
expect(encryptDocumentSummary).toHaveBeenCalledWith(
"A lab report from a clinic listing routine blood values.",
);
// Budget reserved then reconciled at the full reserved amount (charged).
expect(reconcileSpend).toHaveBeenCalledWith("user-1", 5, 5, "2026-07-17");
// Write is scoped to the still-null column so a re-run cannot clobber it.
expect(prisma.inboundDocument.updateMany).toHaveBeenCalledWith(
expect.objectContaining({
where: expect.objectContaining({
id: "doc-1",
userId: "user-1",
summaryEncrypted: null,
}),
data: expect.objectContaining({
summaryEncrypted: expect.any(Uint8Array),
summaryGeneratedAt: expect.any(Date),
}),
}),
);
});
it("SKIPS a document that already has a summary (before reading the opt-in)", async () => {
vi.mocked(prisma.inboundDocument.findFirst).mockResolvedValue({
id: "doc-1",
summaryEncrypted: new Uint8Array([1]),
} as never);
await runDocumentSummaryJob({ userId: "user-1", documentId: "doc-1" });
expect(documentAutoReadEnabled).not.toHaveBeenCalled();
expect(runDocumentSummary).not.toHaveBeenCalled();
expect(prisma.inboundDocument.updateMany).not.toHaveBeenCalled();
});
it("is a graceful no-op when NO provider is configured", async () => {
vi.mocked(resolveDocumentVisionProvider).mockResolvedValue({
chain: [],
pick: null,
} as never);
await runDocumentSummaryJob({ userId: "user-1", documentId: "doc-1" });
expect(reserveBudget).not.toHaveBeenCalled();
expect(runDocumentSummary).not.toHaveBeenCalled();
expect(prisma.inboundDocument.updateMany).not.toHaveBeenCalled();
});
it("refunds the budget and persists nothing when the provider call throws", async () => {
vi.mocked(runDocumentSummary).mockRejectedValueOnce(new Error("boom"));
await runDocumentSummaryJob({ userId: "user-1", documentId: "doc-1" });
// Reservation refunded to zero spend; no write.
expect(reconcileSpend).toHaveBeenCalledWith("user-1", 5, 0, "2026-07-17");
expect(prisma.inboundDocument.updateMany).not.toHaveBeenCalled();
});
});
+243
View File
@@ -0,0 +1,243 @@
/**
* Automatic per-document plain-language summary, enqueued the moment a document
* is stored (`POST /api/documents/inbound`). Fire-and-forget: the upload
* response never blocks on or fails because of it.
*
* Unlike the on-demand summary route (which is transient — nothing persisted),
* THIS path persists a short (3-4 sentence) descriptive summary ENCRYPTED on the
* document row, generated ONCE. It runs ONLY when the user's `documentsAutoAiRead`
* opt-in is ON — that flag is the standing AI-egress consent that removes the
* per-document friction. With the opt-in OFF the job is a strict no-op and the
* on-demand route stays the only way to get a summary.
*
* The gate order mirrors the vault's other AI paths (module gate is enforced at
* the enqueueing upload route): load → opt-in ON → provider resolved (DOCUMENT
* order, local-first) → egress consent re-asserted → budget reserved → generate
* → persist encrypted. Every skip is graceful (no throw, no retry): an already-
* summarised document, no configured provider, a spent budget, or a document the
* picked provider cannot read all resolve to a clean no-op. The summary is
* DESCRIPTIVE ONLY (interpretation boundary G7, enforced by `runDocumentSummary`).
*
* The queue MUST be registered in the maintenance registrar
* (`src/lib/jobs/reminder/register-maintenance.ts`) so pg-boss provisions it.
*/
import { AI_BUDGETS } from "@/lib/ai/ai-budgets";
import {
assertDocumentEgressConsent,
ConsentRequiredError,
} from "@/lib/ai/consent-guard";
import {
buildDateKey,
reconcileSpend,
reserveBudget,
resolveDailyCap,
} from "@/lib/ai/coach/budget";
import {
loadOwnedDocument,
prepareVisionInput,
} from "@/lib/documents/ai-route-support";
import { runDocumentSummary } from "@/lib/documents/describe";
import { documentAutoReadEnabled } from "@/lib/documents/document-settings";
import { resolveDocumentVisionProvider } from "@/lib/documents/provider-order";
import { encryptDocumentSummary } from "@/lib/documents/store";
import { prisma } from "@/lib/db";
import { getGlobalBoss } from "@/lib/jobs/boss-instance";
import { annotate } from "@/lib/logging/context";
export const DOCUMENT_SUMMARY_QUEUE = "document-summary";
/** Serial concurrency — provider calls, kept off the request pool. */
export const DOCUMENT_SUMMARY_CONCURRENCY = 1;
export interface DocumentSummaryPayload {
userId: string;
documentId: string;
enqueuedAt?: string;
}
/**
* Generate + persist ONE document's background summary (owner-scoped). Never
* throws for an expected outcome — every gate that fails resolves to a tagged
* no-op so pg-boss does not retry a document that simply has no provider or a
* user who opted out. Idempotent: a document that already carries a summary is
* skipped, and the final write only lands when the column is still null.
*/
export async function runDocumentSummaryJob(
payload: DocumentSummaryPayload,
): Promise<void> {
const { userId, documentId } = payload;
if (!userId || !documentId) return;
// Idempotency: a document that already has a summary is a cheap no-op (a
// re-enqueue after a duplicate upload, or a boot backfill overlap).
const existing = await prisma.inboundDocument.findFirst({
where: { id: documentId, userId, deletedAt: null },
select: { id: true, summaryEncrypted: true },
});
if (!existing || existing.summaryEncrypted) {
annotate({
action: { name: "documents.summary.autoSkipped" },
meta: { documentId, reason: !existing ? "not-found" : "exists" },
});
return;
}
// The opt-in IS the trigger: when it is OFF the auto path does nothing (the
// on-demand route stays the only way to summarise). Fail-closed on a missing
// row.
if (!(await documentAutoReadEnabled(userId))) {
annotate({
action: { name: "documents.summary.autoSkipped" },
meta: { documentId, reason: "opt-out" },
});
return;
}
// Resolve the DOCUMENT-order vision provider (local-first, codex last). No
// vision-capable provider configured → graceful no-op (unlike the index job,
// there is no local text-layer fallback for a descriptive summary).
const { pick } = await resolveDocumentVisionProvider(userId);
if (!pick) {
annotate({
action: { name: "documents.summary.autoSkipped" },
meta: { documentId, reason: "no-provider" },
});
return;
}
// Re-assert egress consent for the picked provider. A local pick is ungated;
// an external pick is authorised by the `documentsAutoAiRead` opt-in checked
// above (the toggle short-circuits the gate). Belt-and-braces: a consent race
// (opt-out flipped mid-flight) resolves to a no-op, never an egress.
try {
await assertDocumentEgressConsent({
userId,
providerType: pick.providerType,
surface: "insights",
});
} catch (err) {
annotate({
action: { name: "documents.summary.autoSkipped" },
meta: {
documentId,
reason:
err instanceof ConsentRequiredError ? "consent" : "consent-error",
},
});
return;
}
const document = await loadOwnedDocument(userId, documentId);
if (!document) return;
const vision = await prepareVisionInput(document, pick.pdfSupported);
if (!vision.ok) {
// Cannot read this file with the picked provider (decrypt miss, unsupported
// type, or a PDF needing an Anthropic provider). Graceful no-op.
annotate({
action: { name: "documents.summary.autoSkipped" },
meta: { documentId, reason: vision.reason },
});
return;
}
const dateKey = buildDateKey();
const reservation = await reserveBudget(
userId,
AI_BUDGETS.documentSummary.maxTokens,
dateKey,
resolveDailyCap([{ providerType: pick.entry.providerType }]),
);
// Budget exhausted → skip (no local fallback for a summary).
if (!reservation.allowed) {
annotate({
action: { name: "documents.summary.autoSkipped" },
meta: { documentId, reason: "budget" },
});
return;
}
try {
const { summary } = await runDocumentSummary({
provider: pick.entry.instance,
providerType: pick.providerType,
images: vision.images,
documents: vision.documents,
});
await reconcileSpend(
userId,
reservation.reserved,
reservation.reserved,
dateKey,
);
// Persist ENCRYPTED. `updateMany` scoped to `summaryEncrypted: null` so a
// racing writer (a re-enqueue) never overwrites an existing summary — the
// first write wins and a second run is a no-op.
const written = await prisma.inboundDocument.updateMany({
where: {
id: documentId,
userId,
deletedAt: null,
summaryEncrypted: null,
},
data: {
summaryEncrypted: encryptDocumentSummary(summary),
summaryGeneratedAt: new Date(),
},
});
annotate({
action: { name: "documents.summary.autoGenerated" },
meta: {
documentId,
providerType: pick.providerType,
length: summary.length,
persisted: written.count > 0,
},
});
} catch {
// Refund the reservation on a provider miss; the document keeps no summary
// (the on-demand route remains the manual fallback). Never rethrow — a
// transient provider error must not retry-loop or fail the queue.
await reconcileSpend(userId, reservation.reserved, 0, dateKey);
annotate({
action: { name: "documents.summary.autoFailed" },
meta: { documentId, reason: "provider_error" },
});
}
}
/**
* Enqueue a per-document summary job. Coalesced by `singletonKey` per document
* so a duplicate/idempotent re-upload cannot pile up parallel runs. Fire-and-
* forget — a missing boss (worker not up) is a no-op, and a `boss.send` failure
* (a transient DB hiccup) is swallowed to a no-op too, so enqueue can NEVER fail
* an already-stored upload. The summary is a background nicety; the stored
* document is the contract.
*/
export async function enqueueDocumentSummary(
userId: string,
documentId: string,
): Promise<{ enqueued: boolean }> {
const boss = getGlobalBoss();
if (!boss) return { enqueued: false };
const payload: DocumentSummaryPayload = {
userId,
documentId,
enqueuedAt: new Date().toISOString(),
};
try {
const jobId = await boss.send(DOCUMENT_SUMMARY_QUEUE, payload, {
retryLimit: 2,
retryDelay: 30,
retryBackoff: true,
singletonKey: `document-summary|${documentId}`,
});
return { enqueued: Boolean(jobId) };
} catch (err) {
annotate({
action: { name: "documents.summary.enqueueFailed" },
meta: { documentId, reason: err instanceof Error ? err.name : "unknown" },
});
return { enqueued: false };
}
}
@@ -87,6 +87,12 @@ import {
runDocumentThumbnail,
type DocumentThumbnailPayload,
} from "@/lib/jobs/document-thumbnail";
import {
DOCUMENT_SUMMARY_QUEUE,
DOCUMENT_SUMMARY_CONCURRENCY,
runDocumentSummaryJob,
type DocumentSummaryPayload,
} from "@/lib/jobs/document-summary";
import {
DOCUMENT_THUMBNAIL_BACKFILL_QUEUE,
DOCUMENT_THUMBNAIL_BACKFILL_CONCURRENCY,
@@ -344,6 +350,11 @@ const allQueues = [
// thumbnail jobs. Without this entry pg-boss never provisions the queue and
// the boot discovery silently no-ops.
DOCUMENT_THUMBNAIL_BACKFILL_QUEUE,
// Document AI — automatic per-document plain-language summary, enqueued on
// upload. Provider (vision) only, gated on the `documentsAutoAiRead` opt-in +
// egress consent + budget; persists the summary encrypted. Without this entry
// pg-boss never provisions the queue and every upload enqueue silently no-ops.
DOCUMENT_SUMMARY_QUEUE,
];
const schedules: ScheduleEntry[] = [
@@ -768,6 +779,33 @@ export async function registerMaintenanceQueues(
},
);
// Document AI — automatic per-document plain-language summary. Enqueued on
// upload (one job per stored document); runs the vision provider ONLY when
// the `documentsAutoAiRead` opt-in is ON (egress consent + budget gated) and
// persists the summary encrypted. Serial concurrency so the provider call
// never crowds the request pool.
await boss.work<DocumentSummaryPayload>(
DOCUMENT_SUMMARY_QUEUE,
{ localConcurrency: DOCUMENT_SUMMARY_CONCURRENCY },
async (jobs) => {
for (const job of jobs) {
const { userId, documentId } = job.data;
if (!userId || !documentId) continue;
try {
await runDocumentSummaryJob(job.data);
} catch (err) {
recordError();
workerLog(
"error",
`[document-summary] user=${userId} document=${documentId} failed`,
err,
);
throw err;
}
}
},
);
// Document vault — boot-time preview-thumbnail backfill worker. The boot
// discovery sends one per-user job; this handler fans out per-document
// thumbnail jobs for that user's not-yet-thumbnailed documents. Serial
+10 -2
View File
@@ -161,8 +161,16 @@ const inboundDocument = z
});
const inboundDocumentDetail = inboundDocument
.extend({ facts: z.array(extractedFact) })
.meta({ id: "InboundDocumentDetail" });
.extend({
facts: z.array(extractedFact),
summary: z.string().nullable(),
summaryGeneratedAt: z.string().nullable(),
})
.meta({
id: "InboundDocumentDetail",
description:
"A stored document plus its staged facts. `summary` is a short (3-4 sentence) plain-language description of WHAT the document is, generated once in the background right after upload when the `documentsAutoAiRead` opt-in is ON (that flag is also the standing AI-egress consent); it is descriptive only, never a diagnosis, and is null when auto-read is OFF, no provider is configured, or the background summary has not run yet. `summaryGeneratedAt` is when it was generated (null until then).",
});
const listResponse = z
.object({
+9
View File
@@ -292,6 +292,15 @@ export interface InboundDocumentDto {
export interface InboundDocumentDetailDto extends InboundDocumentDto {
facts: ExtractedFactDto[];
/**
* Short (3-4 sentence) plain-language summary of WHAT the document is,
* generated once in the background after upload when the `documentsAutoAiRead`
* opt-in is ON. Null when auto-read is OFF, no provider is configured, or the
* background summary has not run yet. Descriptive only — never a diagnosis.
*/
summary: string | null;
/** When the background summary was generated (ISO 8601), or null. */
summaryGeneratedAt: string | null;
}
// ─── Edit (correction before approval) ─────────────────────────────────────