mirror of
https://github.com/MBombeck/HealthLog.git
synced 2026-11-09 13:01:00 +00:00
Merge pull request #523 from MBombeck/feat/document-auto-summary
feat(documents): persist an auto summary for uploaded documents
This commit is contained in:
19 files changed
+674
-22
No files matched your search
@@ -29085,6 +29085,14 @@ components:
|
||||
type: array
|
||||
items:
|
||||
$ref: "#/components/schemas/ExtractedFact"
|
||||
summary:
|
||||
anyOf:
|
||||
- type: string
|
||||
- type: "null"
|
||||
summaryGeneratedAt:
|
||||
anyOf:
|
||||
- type: string
|
||||
- type: "null"
|
||||
required:
|
||||
- id
|
||||
- kind
|
||||
@@ -29107,7 +29115,14 @@ components:
|
||||
- createdAt
|
||||
- updatedAt
|
||||
- facts
|
||||
- summary
|
||||
- summaryGeneratedAt
|
||||
additionalProperties: false
|
||||
description: A stored document plus its staged facts. `summary` is a short (3-4 sentence) plain-language description of
|
||||
WHAT the document is, generated once in the background right after upload when the `documentsAutoAiRead` opt-in
|
||||
is ON (that flag is also the standing AI-egress consent); it is descriptive only, never a diagnosis, and is null
|
||||
when auto-read is OFF, no provider is configured, or the background summary has not run yet.
|
||||
`summaryGeneratedAt` is when it was generated (null until then).
|
||||
ExtractedFact:
|
||||
type: object
|
||||
properties:
|
||||
|
||||
@@ -8331,6 +8331,10 @@
|
||||
},
|
||||
"detail": {
|
||||
"title": "Dokumentdetails",
|
||||
"summary": {
|
||||
"title": "Zusammenfassung",
|
||||
"pending": "Die Zusammenfassung wird erstellt."
|
||||
},
|
||||
"loadError": "Dieses Dokument konnte nicht geladen werden.",
|
||||
"saveError": "Die Änderung konnte nicht gespeichert werden.",
|
||||
"deleteError": "Das Dokument konnte nicht gelöscht werden.",
|
||||
|
||||
@@ -8331,6 +8331,10 @@
|
||||
},
|
||||
"detail": {
|
||||
"title": "Document details",
|
||||
"summary": {
|
||||
"title": "Summary",
|
||||
"pending": "The summary is being prepared."
|
||||
},
|
||||
"loadError": "Couldn't load this document.",
|
||||
"saveError": "Couldn't save the change.",
|
||||
"deleteError": "Couldn't delete the document.",
|
||||
|
||||
@@ -8331,6 +8331,10 @@
|
||||
},
|
||||
"detail": {
|
||||
"title": "Detalles del documento",
|
||||
"summary": {
|
||||
"title": "Resumen",
|
||||
"pending": "Se está preparando el resumen."
|
||||
},
|
||||
"loadError": "No se pudo cargar este documento.",
|
||||
"saveError": "No se pudo guardar el cambio.",
|
||||
"deleteError": "No se pudo eliminar el documento.",
|
||||
|
||||
@@ -8331,6 +8331,10 @@
|
||||
},
|
||||
"detail": {
|
||||
"title": "Détails du document",
|
||||
"summary": {
|
||||
"title": "Résumé",
|
||||
"pending": "Le résumé est en cours de préparation."
|
||||
},
|
||||
"loadError": "Impossible de charger ce document.",
|
||||
"saveError": "Impossible d'enregistrer la modification.",
|
||||
"deleteError": "Impossible de supprimer le document.",
|
||||
|
||||
@@ -8331,6 +8331,10 @@
|
||||
},
|
||||
"detail": {
|
||||
"title": "Dettagli del documento",
|
||||
"summary": {
|
||||
"title": "Riepilogo",
|
||||
"pending": "Il riepilogo è in preparazione."
|
||||
},
|
||||
"loadError": "Impossibile caricare questo documento.",
|
||||
"saveError": "Impossibile salvare la modifica.",
|
||||
"deleteError": "Impossibile eliminare il documento.",
|
||||
|
||||
@@ -8331,6 +8331,10 @@
|
||||
},
|
||||
"detail": {
|
||||
"title": "Szczegóły dokumentu",
|
||||
"summary": {
|
||||
"title": "Podsumowanie",
|
||||
"pending": "Podsumowanie jest przygotowywane."
|
||||
},
|
||||
"loadError": "Nie udało się wczytać tego dokumentu.",
|
||||
"saveError": "Nie udało się zapisać zmiany.",
|
||||
"deleteError": "Nie udało się usunąć dokumentu.",
|
||||
|
||||
@@ -0,0 +1,23 @@
|
||||
-- Persist a short plain-language summary of an uploaded document.
|
||||
--
|
||||
-- Adds two nullable columns to `inbound_documents`:
|
||||
-- - `summary_encrypted` — AES-256-GCM ciphertext of a 3-4 sentence
|
||||
-- plain-language description of WHAT the document is, generated ONCE in
|
||||
-- the background right after upload when the user's `documents_auto_ai_read`
|
||||
-- opt-in is ON (that flag is also the standing AI-egress consent). NULL
|
||||
-- when auto-read is OFF, no provider is configured, or generation has not
|
||||
-- run yet.
|
||||
-- - `summary_generated_at` — when that background summary was generated
|
||||
-- (NULL until it runs).
|
||||
--
|
||||
-- STORAGE CONVENTION: Bytes (`bytea`), matching the dominant free-text
|
||||
-- encrypted-note convention already in the schema (the `encrypt()` ciphertext
|
||||
-- string stored UTF-8 as `bytea`, the shape the rotation script's
|
||||
-- `rotateBytesColumn` already covers). The column is registered in
|
||||
-- ENCRYPTED_COLUMNS + the rotation script in the same change.
|
||||
--
|
||||
-- ADDITIVE ONLY: existing rows keep NULL summaries; the on-demand transient
|
||||
-- summary route is unchanged.
|
||||
ALTER TABLE "inbound_documents"
|
||||
ADD COLUMN IF NOT EXISTS "summary_encrypted" BYTEA,
|
||||
ADD COLUMN IF NOT EXISTS "summary_generated_at" TIMESTAMP(3);
|
||||
+31
-20
@@ -6061,9 +6061,9 @@ enum ExtractedFactStatus {
|
||||
}
|
||||
|
||||
model InboundDocument {
|
||||
id String @id @default(cuid())
|
||||
userId String @map("user_id")
|
||||
kind InboundDocumentKind @default(OTHER)
|
||||
id String @id @default(cuid())
|
||||
userId String @map("user_id")
|
||||
kind InboundDocumentKind @default(OTHER)
|
||||
/// User-given label. Deliberately plaintext (like `filename`) so the library
|
||||
/// list can ILIKE-search + ORDER BY in SQL. The document BODY stays
|
||||
/// encrypted; only this short, user-authored title is clear-text. Tradeoff:
|
||||
@@ -6071,18 +6071,18 @@ model InboundDocument {
|
||||
/// cleartext at rest / in backups — the same exposure the pre-existing
|
||||
/// plaintext `filename` already carries. Accepted to keep server-side search
|
||||
/// and sort.
|
||||
title String?
|
||||
title String?
|
||||
/// Original filename as uploaded (sanitised, optional).
|
||||
filename String?
|
||||
filename String?
|
||||
/// Magic-byte-sniffed MIME type the document was stored as.
|
||||
mimeType String @map("mime_type")
|
||||
mimeType String @map("mime_type")
|
||||
/// Stored byte length of the original (pre-encryption) document.
|
||||
byteSize Int @map("byte_size")
|
||||
byteSize Int @map("byte_size")
|
||||
/// AES-256-GCM ciphertext of the raw document bytes (base64 of the binary
|
||||
/// → `encrypt()` string → UTF-8 BYTEA), the `*_encrypted` Bytes convention
|
||||
/// shared with the note columns. The original is sensitive and never
|
||||
/// stored in the clear or logged.
|
||||
contentEncrypted Bytes @map("content_encrypted")
|
||||
contentEncrypted Bytes @map("content_encrypted")
|
||||
/// Lowercase-hex SHA-256 of the PLAINTEXT document bytes, computed at
|
||||
/// upload for same-user duplicate detection. Nullable: rows stored before
|
||||
/// the vault release are not backfilled (only new uploads dedupe). A
|
||||
@@ -6091,40 +6091,51 @@ model InboundDocument {
|
||||
/// CREATE UNIQUE INDEX inbound_documents_user_sha_live
|
||||
/// ON inbound_documents (user_id, content_sha256)
|
||||
/// WHERE deleted_at IS NULL AND content_sha256 IS NOT NULL;
|
||||
contentSha256 String? @map("content_sha256") @db.VarChar(64)
|
||||
contentSha256 String? @map("content_sha256") @db.VarChar(64)
|
||||
/// Storage codec of `contentEncrypted`. "base64v1" = legacy string path
|
||||
/// (base64-of-binary → `encrypt()` string → UTF-8 bytes); "binary2" = the
|
||||
/// binary AES-256-GCM layout written by `encryptBytes()` (no base64
|
||||
/// detour). Explicit column, never header-sniffed; the default keeps every
|
||||
/// pre-existing row correct.
|
||||
contentCodec String @default("base64v1") @map("content_codec")
|
||||
status InboundDocumentStatus @default(STORED)
|
||||
contentCodec String @default("base64v1") @map("content_codec")
|
||||
status InboundDocumentStatus @default(STORED)
|
||||
/// Provider type that ran the extraction (e.g. "anthropic", "openai",
|
||||
/// "local"). Diagnostic only. Null on a store-only document.
|
||||
providerType String? @map("provider_type")
|
||||
providerType String? @map("provider_type")
|
||||
/// The document's stated report/collection date (YYYY-MM-DD at UTC), as
|
||||
/// transcribed by the model — never inferred, never user-set.
|
||||
reportDate DateTime? @map("report_date")
|
||||
reportDate DateTime? @map("report_date")
|
||||
/// Effective filing date the library sorts, groups, and range-filters on
|
||||
/// (editable). Distinct from `reportDate`, which is the model-transcribed
|
||||
/// value. On store it defaults to the upload day (now) when the user does
|
||||
/// not supply one, so a stored row's filing date is never NULL and display
|
||||
/// == sort == filter. It stays user-editable and MAY be cleared back to
|
||||
/// NULL via PATCH (the list then sorts such rows last).
|
||||
documentDate DateTime? @map("document_date")
|
||||
documentDate DateTime? @map("document_date")
|
||||
/// Failure reason when status = FAILED (a short code, never the document).
|
||||
errorReason String? @map("error_reason")
|
||||
errorReason String? @map("error_reason")
|
||||
/// AES-256-GCM ciphertext of a short (3-4 sentence) plain-language summary
|
||||
/// of WHAT the document is, generated ONCE in the background right after
|
||||
/// upload when the user's `documentsAutoAiRead` opt-in is ON (that flag is
|
||||
/// also the standing AI-egress consent). Absent when auto-read is OFF, no
|
||||
/// provider is configured, or generation has not run yet. Descriptive only —
|
||||
/// never a diagnosis. `encrypt()`-string-as-UTF-8 BYTEA, the shared Bytes
|
||||
/// codec every other note column uses; PHI, never logged.
|
||||
summaryEncrypted Bytes? @map("summary_encrypted")
|
||||
/// When the background summary was generated (null until it runs). Surfaced
|
||||
/// as meta beside the summary in the document detail view.
|
||||
summaryGeneratedAt DateTime? @map("summary_generated_at")
|
||||
|
||||
facts ExtractedFact[]
|
||||
conditionLinks DocumentConditionLink[]
|
||||
facts ExtractedFact[]
|
||||
conditionLinks DocumentConditionLink[]
|
||||
/// 1:1 blind content-search index (encrypted extracted text + HMAC token
|
||||
/// array). Absent until the document is indexed; cascades on delete.
|
||||
contentIndex DocumentContentIndex?
|
||||
contentIndex DocumentContentIndex?
|
||||
/// 1:1 encrypted preview thumbnail (small JPEG, AES-256-GCM at rest).
|
||||
/// Absent until the background thumbnail job runs; a side table (not a
|
||||
/// column) so the list/detail `omit: { contentEncrypted }` queries never
|
||||
/// drag the blob into the SELECT. Cascades on delete.
|
||||
thumbnail DocumentThumbnail?
|
||||
thumbnail DocumentThumbnail?
|
||||
/// v1.29.x (S7) — coach-conversation attachments (m:n via
|
||||
/// `CoachConversationDocument`). Deleting this document cascades its join rows
|
||||
/// away (its text leaves every conversation's context) but never the
|
||||
@@ -6132,7 +6143,7 @@ model InboundDocument {
|
||||
chatAttachments CoachConversationDocument[]
|
||||
/// share-link memberships (m:n via `ClinicianShareLinkDocument`).
|
||||
/// Cascades: deleting this document removes it from every share it was on.
|
||||
shareLinks ClinicianShareLinkDocument[]
|
||||
shareLinks ClinicianShareLinkDocument[]
|
||||
|
||||
createdAt DateTime @default(now()) @map("created_at")
|
||||
updatedAt DateTime @updatedAt @map("updated_at")
|
||||
|
||||
@@ -540,6 +540,18 @@ async function main() {
|
||||
prisma.extractedFact,
|
||||
),
|
||||
);
|
||||
// The short background document summary. Small `encrypt()`-string-as-UTF-8
|
||||
// Bytes payload (never the multi-megabyte original blob — the select is
|
||||
// scoped to id + this column), so the generic Bytes walk covers it. NULL on
|
||||
// rows uploaded with auto-read OFF / no provider — `rotateBytesColumn` skips
|
||||
// those.
|
||||
results.push(
|
||||
await rotateBytesColumn(
|
||||
"InboundDocument",
|
||||
"summaryEncrypted",
|
||||
prisma.inboundDocument,
|
||||
),
|
||||
);
|
||||
|
||||
// ───── v1.27.22 document content-search index (Bytes text + re-tokenise) ─────
|
||||
// The blind content index carries TWO coupled artefacts under the index key
|
||||
|
||||
@@ -42,6 +42,7 @@ import {
|
||||
type SerialisableDocument,
|
||||
} from "@/lib/documents/store";
|
||||
import { enqueueDocumentIndex } from "@/lib/jobs/document-index";
|
||||
import { enqueueDocumentSummary } from "@/lib/jobs/document-summary";
|
||||
import { enqueueDocumentThumbnail } from "@/lib/jobs/document-thumbnail";
|
||||
import {
|
||||
detectDocumentType,
|
||||
@@ -378,6 +379,13 @@ async function postUpload(request: Request): Promise<Response> {
|
||||
// Only fresh inserts reach here (a duplicate returns early above).
|
||||
void enqueueDocumentThumbnail(user.id, document.id);
|
||||
|
||||
// Summarise the freshly stored document in the background — but ONLY when the
|
||||
// `documentsAutoAiRead` opt-in is ON (the job re-checks it and the egress
|
||||
// consent, and no-ops otherwise). The persisted summary shows on the detail
|
||||
// view. Same fire-and-forget contract: the upload never blocks on or fails
|
||||
// because of it. Only fresh inserts reach here (a duplicate returns early).
|
||||
void enqueueDocumentSummary(user.id, document.id);
|
||||
|
||||
const links = await loadConditionLinks(user.id, [document.id]);
|
||||
return apiSuccess(
|
||||
serialiseDocument(
|
||||
|
||||
@@ -570,6 +570,35 @@ export function DocumentDetailSheet({
|
||||
</div>
|
||||
)}
|
||||
|
||||
{doc.summary ? (
|
||||
// Persisted background summary — generated once at upload when the
|
||||
// auto-read opt-in is on. AI narrative body → foreground content;
|
||||
// the generation date is muted meta (UI-STANDARDS §3).
|
||||
<section
|
||||
className="space-y-1.5"
|
||||
data-slot="document-detail-summary"
|
||||
>
|
||||
<p className="text-sm leading-none font-medium">
|
||||
{t("documents.detail.summary.title")}
|
||||
</p>
|
||||
<p className="text-foreground text-sm">{doc.summary}</p>
|
||||
{doc.summaryGeneratedAt ? (
|
||||
<p className="text-muted-foreground text-xs">
|
||||
{format.date(doc.summaryGeneratedAt)}
|
||||
</p>
|
||||
) : null}
|
||||
</section>
|
||||
) : autoReadEnabled ? (
|
||||
// Auto-read is on but no summary yet (still generating, or the
|
||||
// provider could not read this file) — one calm muted hint.
|
||||
<p
|
||||
className="text-muted-foreground text-xs"
|
||||
data-slot="document-detail-summary-pending"
|
||||
>
|
||||
{t("documents.detail.summary.pending")}
|
||||
</p>
|
||||
) : null}
|
||||
|
||||
<DocumentAiSection
|
||||
aiEnabled={aiEnabled}
|
||||
autoReadEnabled={autoReadEnabled}
|
||||
|
||||
@@ -229,6 +229,12 @@ export const ENCRYPTED_COLUMNS: readonly EncryptedColumn[] = [
|
||||
// document itself rather than persisting as plaintext JSONB.
|
||||
{ model: "ExtractedFact", field: "dataEncrypted", kind: "bytes" },
|
||||
{ model: "ExtractedFact", field: "provenanceEncrypted", kind: "bytes" },
|
||||
// The short plain-language document summary, generated once in the background
|
||||
// after upload. Stored as the `encrypt()`-string-as-UTF-8 Bytes shape (the
|
||||
// same codec the content index + note columns use), so the generic Bytes walk
|
||||
// re-encrypts it. Nullable — pre-summary and opt-out rows are skipped
|
||||
// generically, as for every other nullable Bytes column.
|
||||
{ model: "InboundDocument", field: "summaryEncrypted", kind: "bytes" },
|
||||
|
||||
// ───── v1.27.22 document content-search index (Bytes text) ─────
|
||||
// The normalised extracted text of an indexed document, AES-256-GCM at rest
|
||||
|
||||
@@ -113,6 +113,25 @@ export function decryptThumbnail(buf: Uint8Array): Buffer {
|
||||
return Buffer.from(decryptFromBytes(buf), "base64");
|
||||
}
|
||||
|
||||
/**
|
||||
* Encrypt a document's short plain-language summary into the `Bytes` payload
|
||||
* the schema stores (`InboundDocument.summaryEncrypted`). Uses the shared
|
||||
* AES-256-GCM string codec (`encrypt()` string → UTF-8 bytes), the same shape
|
||||
* `DocumentContentIndex.textEncrypted` uses — so the standard key-rotation walk
|
||||
* (`rotateBytesColumn`) covers it. The summary is descriptive PHI; same posture
|
||||
* (versioned key id, fail-closed decrypt) as every other encrypted column.
|
||||
*/
|
||||
export function encryptDocumentSummary(
|
||||
summary: string,
|
||||
): Uint8Array<ArrayBuffer> {
|
||||
return encryptToBytes(summary);
|
||||
}
|
||||
|
||||
/** Decrypt a stored document summary's `Bytes` payload. Throws on a bad key. */
|
||||
export function decryptDocumentSummary(buf: Uint8Array): string {
|
||||
return decryptFromBytes(buf);
|
||||
}
|
||||
|
||||
/**
|
||||
* Encrypt a staged fact's FHIR-staged payload into the `Bytes` column the
|
||||
* schema stores. The structured clinical values (diagnosis text, lab values,
|
||||
@@ -217,6 +236,18 @@ export function serialiseDocumentDetail(
|
||||
// `factCount` excludes REJECTED facts (a rejected fact is discarded, not part
|
||||
// of the document's tally) so the badge matches the list query.
|
||||
const factCount = facts.filter((f) => f.status !== "REJECTED").length;
|
||||
// Decrypt the persisted background summary for the detail view. A decrypt
|
||||
// failure (a rotated-away legacy key) degrades to no summary rather than
|
||||
// failing the whole detail load — the field is descriptive, not load-bearing,
|
||||
// and the ciphertext is never returned (fail-closed to null).
|
||||
let summary: string | null = null;
|
||||
if (doc.summaryEncrypted && doc.summaryEncrypted.byteLength > 0) {
|
||||
try {
|
||||
summary = decryptDocumentSummary(doc.summaryEncrypted);
|
||||
} catch {
|
||||
summary = null;
|
||||
}
|
||||
}
|
||||
return {
|
||||
...serialiseDocument(
|
||||
doc,
|
||||
@@ -227,5 +258,9 @@ export function serialiseDocumentDetail(
|
||||
hasThumbnail,
|
||||
),
|
||||
facts: facts.map(serialiseFact),
|
||||
summary,
|
||||
summaryGeneratedAt: doc.summaryGeneratedAt
|
||||
? doc.summaryGeneratedAt.toISOString()
|
||||
: null,
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,191 @@
|
||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
|
||||
/**
|
||||
* The auto-summary background job's gating. Pins: the opt-in (`documentsAutoAiRead`)
|
||||
* is the trigger — OFF → strictly no summary (no provider call, no write); ON +
|
||||
* a configured provider → the summary is generated once and persisted ENCRYPTED
|
||||
* (write scoped to `summaryEncrypted: null` so a re-run is a no-op); an already-
|
||||
* summarised document short-circuits before the opt-in is even read; no provider
|
||||
* configured → a graceful no-op; a provider error refunds the budget and persists
|
||||
* nothing.
|
||||
*/
|
||||
|
||||
vi.mock("@/lib/logging/context", () => ({ annotate: vi.fn() }));
|
||||
vi.mock("@/lib/db", () => ({
|
||||
prisma: {
|
||||
inboundDocument: {
|
||||
findFirst: vi.fn(),
|
||||
updateMany: vi.fn(),
|
||||
},
|
||||
},
|
||||
}));
|
||||
vi.mock("@/lib/jobs/boss-instance", () => ({ getGlobalBoss: vi.fn() }));
|
||||
vi.mock("@/lib/documents/document-settings", () => ({
|
||||
documentAutoReadEnabled: vi.fn(),
|
||||
}));
|
||||
vi.mock("@/lib/documents/provider-order", () => ({
|
||||
resolveDocumentVisionProvider: vi.fn(),
|
||||
}));
|
||||
vi.mock("@/lib/documents/ai-route-support", () => ({
|
||||
loadOwnedDocument: vi.fn(),
|
||||
prepareVisionInput: vi.fn(),
|
||||
}));
|
||||
vi.mock("@/lib/documents/describe", () => ({
|
||||
runDocumentSummary: vi.fn(),
|
||||
}));
|
||||
vi.mock("@/lib/documents/store", () => ({
|
||||
encryptDocumentSummary: vi.fn(() => new Uint8Array([9, 9, 9])),
|
||||
}));
|
||||
vi.mock("@/lib/ai/consent-guard", () => {
|
||||
class ConsentRequiredError extends Error {}
|
||||
return {
|
||||
ConsentRequiredError,
|
||||
assertDocumentEgressConsent: vi.fn(),
|
||||
};
|
||||
});
|
||||
vi.mock("@/lib/ai/coach/budget", () => ({
|
||||
buildDateKey: vi.fn(() => "2026-07-17"),
|
||||
reserveBudget: vi.fn(),
|
||||
reconcileSpend: vi.fn().mockResolvedValue(undefined),
|
||||
resolveDailyCap: vi.fn(() => 1000),
|
||||
}));
|
||||
vi.mock("@/lib/ai/ai-budgets", () => ({
|
||||
AI_BUDGETS: { documentSummary: { temperature: 0.3, maxTokens: 600 } },
|
||||
}));
|
||||
|
||||
import { runDocumentSummaryJob } from "../document-summary";
|
||||
import { prisma } from "@/lib/db";
|
||||
import { documentAutoReadEnabled } from "@/lib/documents/document-settings";
|
||||
import { resolveDocumentVisionProvider } from "@/lib/documents/provider-order";
|
||||
import {
|
||||
loadOwnedDocument,
|
||||
prepareVisionInput,
|
||||
} from "@/lib/documents/ai-route-support";
|
||||
import { runDocumentSummary } from "@/lib/documents/describe";
|
||||
import { encryptDocumentSummary } from "@/lib/documents/store";
|
||||
import { assertDocumentEgressConsent } from "@/lib/ai/consent-guard";
|
||||
import { reserveBudget, reconcileSpend } from "@/lib/ai/coach/budget";
|
||||
|
||||
const DOC = {
|
||||
id: "doc-1",
|
||||
kind: "OTHER",
|
||||
contentEncrypted: new Uint8Array([1, 2, 3]),
|
||||
contentCodec: "binary2",
|
||||
mimeType: "application/pdf",
|
||||
status: "STORED",
|
||||
};
|
||||
|
||||
/** An EXTERNAL (Anthropic) document pick — egress, so opt-in-gated. */
|
||||
const PICK = {
|
||||
chain: [{ providerType: "anthropic", instance: {} }],
|
||||
pick: {
|
||||
entry: { providerType: "anthropic", instance: {} },
|
||||
providerType: "anthropic",
|
||||
pdfSupported: true,
|
||||
},
|
||||
};
|
||||
|
||||
function visionOk() {
|
||||
vi.mocked(prepareVisionInput).mockResolvedValue({
|
||||
ok: true,
|
||||
images: [],
|
||||
documents: [{ mediaType: "application/pdf", dataBase64: "AA==" }],
|
||||
} as never);
|
||||
}
|
||||
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks();
|
||||
// Default: a live, not-yet-summarised document.
|
||||
vi.mocked(prisma.inboundDocument.findFirst).mockResolvedValue({
|
||||
id: "doc-1",
|
||||
summaryEncrypted: null,
|
||||
} as never);
|
||||
vi.mocked(prisma.inboundDocument.updateMany).mockResolvedValue({
|
||||
count: 1,
|
||||
} as never);
|
||||
vi.mocked(documentAutoReadEnabled).mockResolvedValue(true);
|
||||
vi.mocked(resolveDocumentVisionProvider).mockResolvedValue(PICK as never);
|
||||
vi.mocked(assertDocumentEgressConsent).mockResolvedValue(undefined);
|
||||
vi.mocked(loadOwnedDocument).mockResolvedValue(DOC as never);
|
||||
visionOk();
|
||||
vi.mocked(runDocumentSummary).mockResolvedValue({
|
||||
summary: "A lab report from a clinic listing routine blood values.",
|
||||
} as never);
|
||||
vi.mocked(reserveBudget).mockResolvedValue({
|
||||
allowed: true,
|
||||
reserved: 5,
|
||||
} as never);
|
||||
});
|
||||
|
||||
describe("runDocumentSummaryJob — gating", () => {
|
||||
it("does NOTHING when the opt-in is OFF (no provider call, no write)", async () => {
|
||||
vi.mocked(documentAutoReadEnabled).mockResolvedValue(false);
|
||||
|
||||
await runDocumentSummaryJob({ userId: "user-1", documentId: "doc-1" });
|
||||
|
||||
expect(resolveDocumentVisionProvider).not.toHaveBeenCalled();
|
||||
expect(runDocumentSummary).not.toHaveBeenCalled();
|
||||
expect(prisma.inboundDocument.updateMany).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("generates and PERSISTS the summary encrypted when opt-in is ON + a provider is configured", async () => {
|
||||
await runDocumentSummaryJob({ userId: "user-1", documentId: "doc-1" });
|
||||
|
||||
expect(runDocumentSummary).toHaveBeenCalledTimes(1);
|
||||
expect(encryptDocumentSummary).toHaveBeenCalledWith(
|
||||
"A lab report from a clinic listing routine blood values.",
|
||||
);
|
||||
// Budget reserved then reconciled at the full reserved amount (charged).
|
||||
expect(reconcileSpend).toHaveBeenCalledWith("user-1", 5, 5, "2026-07-17");
|
||||
// Write is scoped to the still-null column so a re-run cannot clobber it.
|
||||
expect(prisma.inboundDocument.updateMany).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
where: expect.objectContaining({
|
||||
id: "doc-1",
|
||||
userId: "user-1",
|
||||
summaryEncrypted: null,
|
||||
}),
|
||||
data: expect.objectContaining({
|
||||
summaryEncrypted: expect.any(Uint8Array),
|
||||
summaryGeneratedAt: expect.any(Date),
|
||||
}),
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it("SKIPS a document that already has a summary (before reading the opt-in)", async () => {
|
||||
vi.mocked(prisma.inboundDocument.findFirst).mockResolvedValue({
|
||||
id: "doc-1",
|
||||
summaryEncrypted: new Uint8Array([1]),
|
||||
} as never);
|
||||
|
||||
await runDocumentSummaryJob({ userId: "user-1", documentId: "doc-1" });
|
||||
|
||||
expect(documentAutoReadEnabled).not.toHaveBeenCalled();
|
||||
expect(runDocumentSummary).not.toHaveBeenCalled();
|
||||
expect(prisma.inboundDocument.updateMany).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("is a graceful no-op when NO provider is configured", async () => {
|
||||
vi.mocked(resolveDocumentVisionProvider).mockResolvedValue({
|
||||
chain: [],
|
||||
pick: null,
|
||||
} as never);
|
||||
|
||||
await runDocumentSummaryJob({ userId: "user-1", documentId: "doc-1" });
|
||||
|
||||
expect(reserveBudget).not.toHaveBeenCalled();
|
||||
expect(runDocumentSummary).not.toHaveBeenCalled();
|
||||
expect(prisma.inboundDocument.updateMany).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("refunds the budget and persists nothing when the provider call throws", async () => {
|
||||
vi.mocked(runDocumentSummary).mockRejectedValueOnce(new Error("boom"));
|
||||
|
||||
await runDocumentSummaryJob({ userId: "user-1", documentId: "doc-1" });
|
||||
|
||||
// Reservation refunded to zero spend; no write.
|
||||
expect(reconcileSpend).toHaveBeenCalledWith("user-1", 5, 0, "2026-07-17");
|
||||
expect(prisma.inboundDocument.updateMany).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,243 @@
|
||||
/**
|
||||
* Automatic per-document plain-language summary, enqueued the moment a document
|
||||
* is stored (`POST /api/documents/inbound`). Fire-and-forget: the upload
|
||||
* response never blocks on or fails because of it.
|
||||
*
|
||||
* Unlike the on-demand summary route (which is transient — nothing persisted),
|
||||
* THIS path persists a short (3-4 sentence) descriptive summary ENCRYPTED on the
|
||||
* document row, generated ONCE. It runs ONLY when the user's `documentsAutoAiRead`
|
||||
* opt-in is ON — that flag is the standing AI-egress consent that removes the
|
||||
* per-document friction. With the opt-in OFF the job is a strict no-op and the
|
||||
* on-demand route stays the only way to get a summary.
|
||||
*
|
||||
* The gate order mirrors the vault's other AI paths (module gate is enforced at
|
||||
* the enqueueing upload route): load → opt-in ON → provider resolved (DOCUMENT
|
||||
* order, local-first) → egress consent re-asserted → budget reserved → generate
|
||||
* → persist encrypted. Every skip is graceful (no throw, no retry): an already-
|
||||
* summarised document, no configured provider, a spent budget, or a document the
|
||||
* picked provider cannot read all resolve to a clean no-op. The summary is
|
||||
* DESCRIPTIVE ONLY (interpretation boundary G7, enforced by `runDocumentSummary`).
|
||||
*
|
||||
* The queue MUST be registered in the maintenance registrar
|
||||
* (`src/lib/jobs/reminder/register-maintenance.ts`) so pg-boss provisions it.
|
||||
*/
|
||||
import { AI_BUDGETS } from "@/lib/ai/ai-budgets";
|
||||
import {
|
||||
assertDocumentEgressConsent,
|
||||
ConsentRequiredError,
|
||||
} from "@/lib/ai/consent-guard";
|
||||
import {
|
||||
buildDateKey,
|
||||
reconcileSpend,
|
||||
reserveBudget,
|
||||
resolveDailyCap,
|
||||
} from "@/lib/ai/coach/budget";
|
||||
import {
|
||||
loadOwnedDocument,
|
||||
prepareVisionInput,
|
||||
} from "@/lib/documents/ai-route-support";
|
||||
import { runDocumentSummary } from "@/lib/documents/describe";
|
||||
import { documentAutoReadEnabled } from "@/lib/documents/document-settings";
|
||||
import { resolveDocumentVisionProvider } from "@/lib/documents/provider-order";
|
||||
import { encryptDocumentSummary } from "@/lib/documents/store";
|
||||
import { prisma } from "@/lib/db";
|
||||
import { getGlobalBoss } from "@/lib/jobs/boss-instance";
|
||||
import { annotate } from "@/lib/logging/context";
|
||||
|
||||
export const DOCUMENT_SUMMARY_QUEUE = "document-summary";
|
||||
|
||||
/** Serial concurrency — provider calls, kept off the request pool. */
|
||||
export const DOCUMENT_SUMMARY_CONCURRENCY = 1;
|
||||
|
||||
export interface DocumentSummaryPayload {
|
||||
userId: string;
|
||||
documentId: string;
|
||||
enqueuedAt?: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate + persist ONE document's background summary (owner-scoped). Never
|
||||
* throws for an expected outcome — every gate that fails resolves to a tagged
|
||||
* no-op so pg-boss does not retry a document that simply has no provider or a
|
||||
* user who opted out. Idempotent: a document that already carries a summary is
|
||||
* skipped, and the final write only lands when the column is still null.
|
||||
*/
|
||||
export async function runDocumentSummaryJob(
|
||||
payload: DocumentSummaryPayload,
|
||||
): Promise<void> {
|
||||
const { userId, documentId } = payload;
|
||||
if (!userId || !documentId) return;
|
||||
|
||||
// Idempotency: a document that already has a summary is a cheap no-op (a
|
||||
// re-enqueue after a duplicate upload, or a boot backfill overlap).
|
||||
const existing = await prisma.inboundDocument.findFirst({
|
||||
where: { id: documentId, userId, deletedAt: null },
|
||||
select: { id: true, summaryEncrypted: true },
|
||||
});
|
||||
if (!existing || existing.summaryEncrypted) {
|
||||
annotate({
|
||||
action: { name: "documents.summary.autoSkipped" },
|
||||
meta: { documentId, reason: !existing ? "not-found" : "exists" },
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
// The opt-in IS the trigger: when it is OFF the auto path does nothing (the
|
||||
// on-demand route stays the only way to summarise). Fail-closed on a missing
|
||||
// row.
|
||||
if (!(await documentAutoReadEnabled(userId))) {
|
||||
annotate({
|
||||
action: { name: "documents.summary.autoSkipped" },
|
||||
meta: { documentId, reason: "opt-out" },
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
// Resolve the DOCUMENT-order vision provider (local-first, codex last). No
|
||||
// vision-capable provider configured → graceful no-op (unlike the index job,
|
||||
// there is no local text-layer fallback for a descriptive summary).
|
||||
const { pick } = await resolveDocumentVisionProvider(userId);
|
||||
if (!pick) {
|
||||
annotate({
|
||||
action: { name: "documents.summary.autoSkipped" },
|
||||
meta: { documentId, reason: "no-provider" },
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
// Re-assert egress consent for the picked provider. A local pick is ungated;
|
||||
// an external pick is authorised by the `documentsAutoAiRead` opt-in checked
|
||||
// above (the toggle short-circuits the gate). Belt-and-braces: a consent race
|
||||
// (opt-out flipped mid-flight) resolves to a no-op, never an egress.
|
||||
try {
|
||||
await assertDocumentEgressConsent({
|
||||
userId,
|
||||
providerType: pick.providerType,
|
||||
surface: "insights",
|
||||
});
|
||||
} catch (err) {
|
||||
annotate({
|
||||
action: { name: "documents.summary.autoSkipped" },
|
||||
meta: {
|
||||
documentId,
|
||||
reason:
|
||||
err instanceof ConsentRequiredError ? "consent" : "consent-error",
|
||||
},
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
const document = await loadOwnedDocument(userId, documentId);
|
||||
if (!document) return;
|
||||
|
||||
const vision = await prepareVisionInput(document, pick.pdfSupported);
|
||||
if (!vision.ok) {
|
||||
// Cannot read this file with the picked provider (decrypt miss, unsupported
|
||||
// type, or a PDF needing an Anthropic provider). Graceful no-op.
|
||||
annotate({
|
||||
action: { name: "documents.summary.autoSkipped" },
|
||||
meta: { documentId, reason: vision.reason },
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
const dateKey = buildDateKey();
|
||||
const reservation = await reserveBudget(
|
||||
userId,
|
||||
AI_BUDGETS.documentSummary.maxTokens,
|
||||
dateKey,
|
||||
resolveDailyCap([{ providerType: pick.entry.providerType }]),
|
||||
);
|
||||
// Budget exhausted → skip (no local fallback for a summary).
|
||||
if (!reservation.allowed) {
|
||||
annotate({
|
||||
action: { name: "documents.summary.autoSkipped" },
|
||||
meta: { documentId, reason: "budget" },
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
const { summary } = await runDocumentSummary({
|
||||
provider: pick.entry.instance,
|
||||
providerType: pick.providerType,
|
||||
images: vision.images,
|
||||
documents: vision.documents,
|
||||
});
|
||||
await reconcileSpend(
|
||||
userId,
|
||||
reservation.reserved,
|
||||
reservation.reserved,
|
||||
dateKey,
|
||||
);
|
||||
// Persist ENCRYPTED. `updateMany` scoped to `summaryEncrypted: null` so a
|
||||
// racing writer (a re-enqueue) never overwrites an existing summary — the
|
||||
// first write wins and a second run is a no-op.
|
||||
const written = await prisma.inboundDocument.updateMany({
|
||||
where: {
|
||||
id: documentId,
|
||||
userId,
|
||||
deletedAt: null,
|
||||
summaryEncrypted: null,
|
||||
},
|
||||
data: {
|
||||
summaryEncrypted: encryptDocumentSummary(summary),
|
||||
summaryGeneratedAt: new Date(),
|
||||
},
|
||||
});
|
||||
annotate({
|
||||
action: { name: "documents.summary.autoGenerated" },
|
||||
meta: {
|
||||
documentId,
|
||||
providerType: pick.providerType,
|
||||
length: summary.length,
|
||||
persisted: written.count > 0,
|
||||
},
|
||||
});
|
||||
} catch {
|
||||
// Refund the reservation on a provider miss; the document keeps no summary
|
||||
// (the on-demand route remains the manual fallback). Never rethrow — a
|
||||
// transient provider error must not retry-loop or fail the queue.
|
||||
await reconcileSpend(userId, reservation.reserved, 0, dateKey);
|
||||
annotate({
|
||||
action: { name: "documents.summary.autoFailed" },
|
||||
meta: { documentId, reason: "provider_error" },
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Enqueue a per-document summary job. Coalesced by `singletonKey` per document
|
||||
* so a duplicate/idempotent re-upload cannot pile up parallel runs. Fire-and-
|
||||
* forget — a missing boss (worker not up) is a no-op, and a `boss.send` failure
|
||||
* (a transient DB hiccup) is swallowed to a no-op too, so enqueue can NEVER fail
|
||||
* an already-stored upload. The summary is a background nicety; the stored
|
||||
* document is the contract.
|
||||
*/
|
||||
export async function enqueueDocumentSummary(
|
||||
userId: string,
|
||||
documentId: string,
|
||||
): Promise<{ enqueued: boolean }> {
|
||||
const boss = getGlobalBoss();
|
||||
if (!boss) return { enqueued: false };
|
||||
const payload: DocumentSummaryPayload = {
|
||||
userId,
|
||||
documentId,
|
||||
enqueuedAt: new Date().toISOString(),
|
||||
};
|
||||
try {
|
||||
const jobId = await boss.send(DOCUMENT_SUMMARY_QUEUE, payload, {
|
||||
retryLimit: 2,
|
||||
retryDelay: 30,
|
||||
retryBackoff: true,
|
||||
singletonKey: `document-summary|${documentId}`,
|
||||
});
|
||||
return { enqueued: Boolean(jobId) };
|
||||
} catch (err) {
|
||||
annotate({
|
||||
action: { name: "documents.summary.enqueueFailed" },
|
||||
meta: { documentId, reason: err instanceof Error ? err.name : "unknown" },
|
||||
});
|
||||
return { enqueued: false };
|
||||
}
|
||||
}
|
||||
@@ -87,6 +87,12 @@ import {
|
||||
runDocumentThumbnail,
|
||||
type DocumentThumbnailPayload,
|
||||
} from "@/lib/jobs/document-thumbnail";
|
||||
import {
|
||||
DOCUMENT_SUMMARY_QUEUE,
|
||||
DOCUMENT_SUMMARY_CONCURRENCY,
|
||||
runDocumentSummaryJob,
|
||||
type DocumentSummaryPayload,
|
||||
} from "@/lib/jobs/document-summary";
|
||||
import {
|
||||
DOCUMENT_THUMBNAIL_BACKFILL_QUEUE,
|
||||
DOCUMENT_THUMBNAIL_BACKFILL_CONCURRENCY,
|
||||
@@ -344,6 +350,11 @@ const allQueues = [
|
||||
// thumbnail jobs. Without this entry pg-boss never provisions the queue and
|
||||
// the boot discovery silently no-ops.
|
||||
DOCUMENT_THUMBNAIL_BACKFILL_QUEUE,
|
||||
// Document AI — automatic per-document plain-language summary, enqueued on
|
||||
// upload. Provider (vision) only, gated on the `documentsAutoAiRead` opt-in +
|
||||
// egress consent + budget; persists the summary encrypted. Without this entry
|
||||
// pg-boss never provisions the queue and every upload enqueue silently no-ops.
|
||||
DOCUMENT_SUMMARY_QUEUE,
|
||||
];
|
||||
|
||||
const schedules: ScheduleEntry[] = [
|
||||
@@ -768,6 +779,33 @@ export async function registerMaintenanceQueues(
|
||||
},
|
||||
);
|
||||
|
||||
// Document AI — automatic per-document plain-language summary. Enqueued on
|
||||
// upload (one job per stored document); runs the vision provider ONLY when
|
||||
// the `documentsAutoAiRead` opt-in is ON (egress consent + budget gated) and
|
||||
// persists the summary encrypted. Serial concurrency so the provider call
|
||||
// never crowds the request pool.
|
||||
await boss.work<DocumentSummaryPayload>(
|
||||
DOCUMENT_SUMMARY_QUEUE,
|
||||
{ localConcurrency: DOCUMENT_SUMMARY_CONCURRENCY },
|
||||
async (jobs) => {
|
||||
for (const job of jobs) {
|
||||
const { userId, documentId } = job.data;
|
||||
if (!userId || !documentId) continue;
|
||||
try {
|
||||
await runDocumentSummaryJob(job.data);
|
||||
} catch (err) {
|
||||
recordError();
|
||||
workerLog(
|
||||
"error",
|
||||
`[document-summary] user=${userId} document=${documentId} failed`,
|
||||
err,
|
||||
);
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
// Document vault — boot-time preview-thumbnail backfill worker. The boot
|
||||
// discovery sends one per-user job; this handler fans out per-document
|
||||
// thumbnail jobs for that user's not-yet-thumbnailed documents. Serial
|
||||
|
||||
@@ -161,8 +161,16 @@ const inboundDocument = z
|
||||
});
|
||||
|
||||
const inboundDocumentDetail = inboundDocument
|
||||
.extend({ facts: z.array(extractedFact) })
|
||||
.meta({ id: "InboundDocumentDetail" });
|
||||
.extend({
|
||||
facts: z.array(extractedFact),
|
||||
summary: z.string().nullable(),
|
||||
summaryGeneratedAt: z.string().nullable(),
|
||||
})
|
||||
.meta({
|
||||
id: "InboundDocumentDetail",
|
||||
description:
|
||||
"A stored document plus its staged facts. `summary` is a short (3-4 sentence) plain-language description of WHAT the document is, generated once in the background right after upload when the `documentsAutoAiRead` opt-in is ON (that flag is also the standing AI-egress consent); it is descriptive only, never a diagnosis, and is null when auto-read is OFF, no provider is configured, or the background summary has not run yet. `summaryGeneratedAt` is when it was generated (null until then).",
|
||||
});
|
||||
|
||||
const listResponse = z
|
||||
.object({
|
||||
|
||||
@@ -292,6 +292,15 @@ export interface InboundDocumentDto {
|
||||
|
||||
export interface InboundDocumentDetailDto extends InboundDocumentDto {
|
||||
facts: ExtractedFactDto[];
|
||||
/**
|
||||
* Short (3-4 sentence) plain-language summary of WHAT the document is,
|
||||
* generated once in the background after upload when the `documentsAutoAiRead`
|
||||
* opt-in is ON. Null when auto-read is OFF, no provider is configured, or the
|
||||
* background summary has not run yet. Descriptive only — never a diagnosis.
|
||||
*/
|
||||
summary: string | null;
|
||||
/** When the background summary was generated (ISO 8601), or null. */
|
||||
summaryGeneratedAt: string | null;
|
||||
}
|
||||
|
||||
// ─── Edit (correction before approval) ─────────────────────────────────────
|
||||
|
||||
Reference in new issue
Block a user