mirror of
https://github.com/MBombeck/HealthLog.git
synced 2026-11-13 22:32:04 +00:00
Merge pull request #523 from MBombeck/feat/document-auto-summary
feat(documents): persist an auto summary for uploaded documents
This commit is contained in:
19 files changed
+674
-22
No files matched your search
@@ -42,6 +42,7 @@ import {
|
||||
type SerialisableDocument,
|
||||
} from "@/lib/documents/store";
|
||||
import { enqueueDocumentIndex } from "@/lib/jobs/document-index";
|
||||
import { enqueueDocumentSummary } from "@/lib/jobs/document-summary";
|
||||
import { enqueueDocumentThumbnail } from "@/lib/jobs/document-thumbnail";
|
||||
import {
|
||||
detectDocumentType,
|
||||
@@ -378,6 +379,13 @@ async function postUpload(request: Request): Promise<Response> {
|
||||
// Only fresh inserts reach here (a duplicate returns early above).
|
||||
void enqueueDocumentThumbnail(user.id, document.id);
|
||||
|
||||
// Summarise the freshly stored document in the background — but ONLY when the
|
||||
// `documentsAutoAiRead` opt-in is ON (the job re-checks it and the egress
|
||||
// consent, and no-ops otherwise). The persisted summary shows on the detail
|
||||
// view. Same fire-and-forget contract: the upload never blocks on or fails
|
||||
// because of it. Only fresh inserts reach here (a duplicate returns early).
|
||||
void enqueueDocumentSummary(user.id, document.id);
|
||||
|
||||
const links = await loadConditionLinks(user.id, [document.id]);
|
||||
return apiSuccess(
|
||||
serialiseDocument(
|
||||
|
||||
@@ -570,6 +570,35 @@ export function DocumentDetailSheet({
|
||||
</div>
|
||||
)}
|
||||
|
||||
{doc.summary ? (
|
||||
// Persisted background summary — generated once at upload when the
|
||||
// auto-read opt-in is on. AI narrative body → foreground content;
|
||||
// the generation date is muted meta (UI-STANDARDS §3).
|
||||
<section
|
||||
className="space-y-1.5"
|
||||
data-slot="document-detail-summary"
|
||||
>
|
||||
<p className="text-sm leading-none font-medium">
|
||||
{t("documents.detail.summary.title")}
|
||||
</p>
|
||||
<p className="text-foreground text-sm">{doc.summary}</p>
|
||||
{doc.summaryGeneratedAt ? (
|
||||
<p className="text-muted-foreground text-xs">
|
||||
{format.date(doc.summaryGeneratedAt)}
|
||||
</p>
|
||||
) : null}
|
||||
</section>
|
||||
) : autoReadEnabled ? (
|
||||
// Auto-read is on but no summary yet (still generating, or the
|
||||
// provider could not read this file) — one calm muted hint.
|
||||
<p
|
||||
className="text-muted-foreground text-xs"
|
||||
data-slot="document-detail-summary-pending"
|
||||
>
|
||||
{t("documents.detail.summary.pending")}
|
||||
</p>
|
||||
) : null}
|
||||
|
||||
<DocumentAiSection
|
||||
aiEnabled={aiEnabled}
|
||||
autoReadEnabled={autoReadEnabled}
|
||||
|
||||
@@ -229,6 +229,12 @@ export const ENCRYPTED_COLUMNS: readonly EncryptedColumn[] = [
|
||||
// document itself rather than persisting as plaintext JSONB.
|
||||
{ model: "ExtractedFact", field: "dataEncrypted", kind: "bytes" },
|
||||
{ model: "ExtractedFact", field: "provenanceEncrypted", kind: "bytes" },
|
||||
// The short plain-language document summary, generated once in the background
|
||||
// after upload. Stored as the `encrypt()`-string-as-UTF-8 Bytes shape (the
|
||||
// same codec the content index + note columns use), so the generic Bytes walk
|
||||
// re-encrypts it. Nullable — pre-summary and opt-out rows are skipped
|
||||
// generically, as for every other nullable Bytes column.
|
||||
{ model: "InboundDocument", field: "summaryEncrypted", kind: "bytes" },
|
||||
|
||||
// ───── v1.27.22 document content-search index (Bytes text) ─────
|
||||
// The normalised extracted text of an indexed document, AES-256-GCM at rest
|
||||
|
||||
@@ -113,6 +113,25 @@ export function decryptThumbnail(buf: Uint8Array): Buffer {
|
||||
return Buffer.from(decryptFromBytes(buf), "base64");
|
||||
}
|
||||
|
||||
/**
|
||||
* Encrypt a document's short plain-language summary into the `Bytes` payload
|
||||
* the schema stores (`InboundDocument.summaryEncrypted`). Uses the shared
|
||||
* AES-256-GCM string codec (`encrypt()` string → UTF-8 bytes), the same shape
|
||||
* `DocumentContentIndex.textEncrypted` uses — so the standard key-rotation walk
|
||||
* (`rotateBytesColumn`) covers it. The summary is descriptive PHI; same posture
|
||||
* (versioned key id, fail-closed decrypt) as every other encrypted column.
|
||||
*/
|
||||
export function encryptDocumentSummary(
|
||||
summary: string,
|
||||
): Uint8Array<ArrayBuffer> {
|
||||
return encryptToBytes(summary);
|
||||
}
|
||||
|
||||
/** Decrypt a stored document summary's `Bytes` payload. Throws on a bad key. */
|
||||
export function decryptDocumentSummary(buf: Uint8Array): string {
|
||||
return decryptFromBytes(buf);
|
||||
}
|
||||
|
||||
/**
|
||||
* Encrypt a staged fact's FHIR-staged payload into the `Bytes` column the
|
||||
* schema stores. The structured clinical values (diagnosis text, lab values,
|
||||
@@ -217,6 +236,18 @@ export function serialiseDocumentDetail(
|
||||
// `factCount` excludes REJECTED facts (a rejected fact is discarded, not part
|
||||
// of the document's tally) so the badge matches the list query.
|
||||
const factCount = facts.filter((f) => f.status !== "REJECTED").length;
|
||||
// Decrypt the persisted background summary for the detail view. A decrypt
|
||||
// failure (a rotated-away legacy key) degrades to no summary rather than
|
||||
// failing the whole detail load — the field is descriptive, not load-bearing,
|
||||
// and the ciphertext is never returned (fail-closed to null).
|
||||
let summary: string | null = null;
|
||||
if (doc.summaryEncrypted && doc.summaryEncrypted.byteLength > 0) {
|
||||
try {
|
||||
summary = decryptDocumentSummary(doc.summaryEncrypted);
|
||||
} catch {
|
||||
summary = null;
|
||||
}
|
||||
}
|
||||
return {
|
||||
...serialiseDocument(
|
||||
doc,
|
||||
@@ -227,5 +258,9 @@ export function serialiseDocumentDetail(
|
||||
hasThumbnail,
|
||||
),
|
||||
facts: facts.map(serialiseFact),
|
||||
summary,
|
||||
summaryGeneratedAt: doc.summaryGeneratedAt
|
||||
? doc.summaryGeneratedAt.toISOString()
|
||||
: null,
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,191 @@
|
||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
|
||||
/**
|
||||
* The auto-summary background job's gating. Pins: the opt-in (`documentsAutoAiRead`)
|
||||
* is the trigger — OFF → strictly no summary (no provider call, no write); ON +
|
||||
* a configured provider → the summary is generated once and persisted ENCRYPTED
|
||||
* (write scoped to `summaryEncrypted: null` so a re-run is a no-op); an already-
|
||||
* summarised document short-circuits before the opt-in is even read; no provider
|
||||
* configured → a graceful no-op; a provider error refunds the budget and persists
|
||||
* nothing.
|
||||
*/
|
||||
|
||||
vi.mock("@/lib/logging/context", () => ({ annotate: vi.fn() }));
|
||||
vi.mock("@/lib/db", () => ({
|
||||
prisma: {
|
||||
inboundDocument: {
|
||||
findFirst: vi.fn(),
|
||||
updateMany: vi.fn(),
|
||||
},
|
||||
},
|
||||
}));
|
||||
vi.mock("@/lib/jobs/boss-instance", () => ({ getGlobalBoss: vi.fn() }));
|
||||
vi.mock("@/lib/documents/document-settings", () => ({
|
||||
documentAutoReadEnabled: vi.fn(),
|
||||
}));
|
||||
vi.mock("@/lib/documents/provider-order", () => ({
|
||||
resolveDocumentVisionProvider: vi.fn(),
|
||||
}));
|
||||
vi.mock("@/lib/documents/ai-route-support", () => ({
|
||||
loadOwnedDocument: vi.fn(),
|
||||
prepareVisionInput: vi.fn(),
|
||||
}));
|
||||
vi.mock("@/lib/documents/describe", () => ({
|
||||
runDocumentSummary: vi.fn(),
|
||||
}));
|
||||
vi.mock("@/lib/documents/store", () => ({
|
||||
encryptDocumentSummary: vi.fn(() => new Uint8Array([9, 9, 9])),
|
||||
}));
|
||||
vi.mock("@/lib/ai/consent-guard", () => {
|
||||
class ConsentRequiredError extends Error {}
|
||||
return {
|
||||
ConsentRequiredError,
|
||||
assertDocumentEgressConsent: vi.fn(),
|
||||
};
|
||||
});
|
||||
vi.mock("@/lib/ai/coach/budget", () => ({
|
||||
buildDateKey: vi.fn(() => "2026-07-17"),
|
||||
reserveBudget: vi.fn(),
|
||||
reconcileSpend: vi.fn().mockResolvedValue(undefined),
|
||||
resolveDailyCap: vi.fn(() => 1000),
|
||||
}));
|
||||
vi.mock("@/lib/ai/ai-budgets", () => ({
|
||||
AI_BUDGETS: { documentSummary: { temperature: 0.3, maxTokens: 600 } },
|
||||
}));
|
||||
|
||||
import { runDocumentSummaryJob } from "../document-summary";
|
||||
import { prisma } from "@/lib/db";
|
||||
import { documentAutoReadEnabled } from "@/lib/documents/document-settings";
|
||||
import { resolveDocumentVisionProvider } from "@/lib/documents/provider-order";
|
||||
import {
|
||||
loadOwnedDocument,
|
||||
prepareVisionInput,
|
||||
} from "@/lib/documents/ai-route-support";
|
||||
import { runDocumentSummary } from "@/lib/documents/describe";
|
||||
import { encryptDocumentSummary } from "@/lib/documents/store";
|
||||
import { assertDocumentEgressConsent } from "@/lib/ai/consent-guard";
|
||||
import { reserveBudget, reconcileSpend } from "@/lib/ai/coach/budget";
|
||||
|
||||
const DOC = {
|
||||
id: "doc-1",
|
||||
kind: "OTHER",
|
||||
contentEncrypted: new Uint8Array([1, 2, 3]),
|
||||
contentCodec: "binary2",
|
||||
mimeType: "application/pdf",
|
||||
status: "STORED",
|
||||
};
|
||||
|
||||
/** An EXTERNAL (Anthropic) document pick — egress, so opt-in-gated. */
|
||||
const PICK = {
|
||||
chain: [{ providerType: "anthropic", instance: {} }],
|
||||
pick: {
|
||||
entry: { providerType: "anthropic", instance: {} },
|
||||
providerType: "anthropic",
|
||||
pdfSupported: true,
|
||||
},
|
||||
};
|
||||
|
||||
function visionOk() {
|
||||
vi.mocked(prepareVisionInput).mockResolvedValue({
|
||||
ok: true,
|
||||
images: [],
|
||||
documents: [{ mediaType: "application/pdf", dataBase64: "AA==" }],
|
||||
} as never);
|
||||
}
|
||||
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks();
|
||||
// Default: a live, not-yet-summarised document.
|
||||
vi.mocked(prisma.inboundDocument.findFirst).mockResolvedValue({
|
||||
id: "doc-1",
|
||||
summaryEncrypted: null,
|
||||
} as never);
|
||||
vi.mocked(prisma.inboundDocument.updateMany).mockResolvedValue({
|
||||
count: 1,
|
||||
} as never);
|
||||
vi.mocked(documentAutoReadEnabled).mockResolvedValue(true);
|
||||
vi.mocked(resolveDocumentVisionProvider).mockResolvedValue(PICK as never);
|
||||
vi.mocked(assertDocumentEgressConsent).mockResolvedValue(undefined);
|
||||
vi.mocked(loadOwnedDocument).mockResolvedValue(DOC as never);
|
||||
visionOk();
|
||||
vi.mocked(runDocumentSummary).mockResolvedValue({
|
||||
summary: "A lab report from a clinic listing routine blood values.",
|
||||
} as never);
|
||||
vi.mocked(reserveBudget).mockResolvedValue({
|
||||
allowed: true,
|
||||
reserved: 5,
|
||||
} as never);
|
||||
});
|
||||
|
||||
describe("runDocumentSummaryJob — gating", () => {
|
||||
it("does NOTHING when the opt-in is OFF (no provider call, no write)", async () => {
|
||||
vi.mocked(documentAutoReadEnabled).mockResolvedValue(false);
|
||||
|
||||
await runDocumentSummaryJob({ userId: "user-1", documentId: "doc-1" });
|
||||
|
||||
expect(resolveDocumentVisionProvider).not.toHaveBeenCalled();
|
||||
expect(runDocumentSummary).not.toHaveBeenCalled();
|
||||
expect(prisma.inboundDocument.updateMany).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("generates and PERSISTS the summary encrypted when opt-in is ON + a provider is configured", async () => {
|
||||
await runDocumentSummaryJob({ userId: "user-1", documentId: "doc-1" });
|
||||
|
||||
expect(runDocumentSummary).toHaveBeenCalledTimes(1);
|
||||
expect(encryptDocumentSummary).toHaveBeenCalledWith(
|
||||
"A lab report from a clinic listing routine blood values.",
|
||||
);
|
||||
// Budget reserved then reconciled at the full reserved amount (charged).
|
||||
expect(reconcileSpend).toHaveBeenCalledWith("user-1", 5, 5, "2026-07-17");
|
||||
// Write is scoped to the still-null column so a re-run cannot clobber it.
|
||||
expect(prisma.inboundDocument.updateMany).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
where: expect.objectContaining({
|
||||
id: "doc-1",
|
||||
userId: "user-1",
|
||||
summaryEncrypted: null,
|
||||
}),
|
||||
data: expect.objectContaining({
|
||||
summaryEncrypted: expect.any(Uint8Array),
|
||||
summaryGeneratedAt: expect.any(Date),
|
||||
}),
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it("SKIPS a document that already has a summary (before reading the opt-in)", async () => {
|
||||
vi.mocked(prisma.inboundDocument.findFirst).mockResolvedValue({
|
||||
id: "doc-1",
|
||||
summaryEncrypted: new Uint8Array([1]),
|
||||
} as never);
|
||||
|
||||
await runDocumentSummaryJob({ userId: "user-1", documentId: "doc-1" });
|
||||
|
||||
expect(documentAutoReadEnabled).not.toHaveBeenCalled();
|
||||
expect(runDocumentSummary).not.toHaveBeenCalled();
|
||||
expect(prisma.inboundDocument.updateMany).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("is a graceful no-op when NO provider is configured", async () => {
|
||||
vi.mocked(resolveDocumentVisionProvider).mockResolvedValue({
|
||||
chain: [],
|
||||
pick: null,
|
||||
} as never);
|
||||
|
||||
await runDocumentSummaryJob({ userId: "user-1", documentId: "doc-1" });
|
||||
|
||||
expect(reserveBudget).not.toHaveBeenCalled();
|
||||
expect(runDocumentSummary).not.toHaveBeenCalled();
|
||||
expect(prisma.inboundDocument.updateMany).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("refunds the budget and persists nothing when the provider call throws", async () => {
|
||||
vi.mocked(runDocumentSummary).mockRejectedValueOnce(new Error("boom"));
|
||||
|
||||
await runDocumentSummaryJob({ userId: "user-1", documentId: "doc-1" });
|
||||
|
||||
// Reservation refunded to zero spend; no write.
|
||||
expect(reconcileSpend).toHaveBeenCalledWith("user-1", 5, 0, "2026-07-17");
|
||||
expect(prisma.inboundDocument.updateMany).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,243 @@
|
||||
/**
|
||||
* Automatic per-document plain-language summary, enqueued the moment a document
|
||||
* is stored (`POST /api/documents/inbound`). Fire-and-forget: the upload
|
||||
* response never blocks on or fails because of it.
|
||||
*
|
||||
* Unlike the on-demand summary route (which is transient — nothing persisted),
|
||||
* THIS path persists a short (3-4 sentence) descriptive summary ENCRYPTED on the
|
||||
* document row, generated ONCE. It runs ONLY when the user's `documentsAutoAiRead`
|
||||
* opt-in is ON — that flag is the standing AI-egress consent that removes the
|
||||
* per-document friction. With the opt-in OFF the job is a strict no-op and the
|
||||
* on-demand route stays the only way to get a summary.
|
||||
*
|
||||
* The gate order mirrors the vault's other AI paths (module gate is enforced at
|
||||
* the enqueueing upload route): load → opt-in ON → provider resolved (DOCUMENT
|
||||
* order, local-first) → egress consent re-asserted → budget reserved → generate
|
||||
* → persist encrypted. Every skip is graceful (no throw, no retry): an already-
|
||||
* summarised document, no configured provider, a spent budget, or a document the
|
||||
* picked provider cannot read all resolve to a clean no-op. The summary is
|
||||
* DESCRIPTIVE ONLY (interpretation boundary G7, enforced by `runDocumentSummary`).
|
||||
*
|
||||
* The queue MUST be registered in the maintenance registrar
|
||||
* (`src/lib/jobs/reminder/register-maintenance.ts`) so pg-boss provisions it.
|
||||
*/
|
||||
import { AI_BUDGETS } from "@/lib/ai/ai-budgets";
|
||||
import {
|
||||
assertDocumentEgressConsent,
|
||||
ConsentRequiredError,
|
||||
} from "@/lib/ai/consent-guard";
|
||||
import {
|
||||
buildDateKey,
|
||||
reconcileSpend,
|
||||
reserveBudget,
|
||||
resolveDailyCap,
|
||||
} from "@/lib/ai/coach/budget";
|
||||
import {
|
||||
loadOwnedDocument,
|
||||
prepareVisionInput,
|
||||
} from "@/lib/documents/ai-route-support";
|
||||
import { runDocumentSummary } from "@/lib/documents/describe";
|
||||
import { documentAutoReadEnabled } from "@/lib/documents/document-settings";
|
||||
import { resolveDocumentVisionProvider } from "@/lib/documents/provider-order";
|
||||
import { encryptDocumentSummary } from "@/lib/documents/store";
|
||||
import { prisma } from "@/lib/db";
|
||||
import { getGlobalBoss } from "@/lib/jobs/boss-instance";
|
||||
import { annotate } from "@/lib/logging/context";
|
||||
|
||||
export const DOCUMENT_SUMMARY_QUEUE = "document-summary";
|
||||
|
||||
/** Serial concurrency — provider calls, kept off the request pool. */
|
||||
export const DOCUMENT_SUMMARY_CONCURRENCY = 1;
|
||||
|
||||
export interface DocumentSummaryPayload {
|
||||
userId: string;
|
||||
documentId: string;
|
||||
enqueuedAt?: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate + persist ONE document's background summary (owner-scoped). Never
|
||||
* throws for an expected outcome — every gate that fails resolves to a tagged
|
||||
* no-op so pg-boss does not retry a document that simply has no provider or a
|
||||
* user who opted out. Idempotent: a document that already carries a summary is
|
||||
* skipped, and the final write only lands when the column is still null.
|
||||
*/
|
||||
export async function runDocumentSummaryJob(
|
||||
payload: DocumentSummaryPayload,
|
||||
): Promise<void> {
|
||||
const { userId, documentId } = payload;
|
||||
if (!userId || !documentId) return;
|
||||
|
||||
// Idempotency: a document that already has a summary is a cheap no-op (a
|
||||
// re-enqueue after a duplicate upload, or a boot backfill overlap).
|
||||
const existing = await prisma.inboundDocument.findFirst({
|
||||
where: { id: documentId, userId, deletedAt: null },
|
||||
select: { id: true, summaryEncrypted: true },
|
||||
});
|
||||
if (!existing || existing.summaryEncrypted) {
|
||||
annotate({
|
||||
action: { name: "documents.summary.autoSkipped" },
|
||||
meta: { documentId, reason: !existing ? "not-found" : "exists" },
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
// The opt-in IS the trigger: when it is OFF the auto path does nothing (the
|
||||
// on-demand route stays the only way to summarise). Fail-closed on a missing
|
||||
// row.
|
||||
if (!(await documentAutoReadEnabled(userId))) {
|
||||
annotate({
|
||||
action: { name: "documents.summary.autoSkipped" },
|
||||
meta: { documentId, reason: "opt-out" },
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
// Resolve the DOCUMENT-order vision provider (local-first, codex last). No
|
||||
// vision-capable provider configured → graceful no-op (unlike the index job,
|
||||
// there is no local text-layer fallback for a descriptive summary).
|
||||
const { pick } = await resolveDocumentVisionProvider(userId);
|
||||
if (!pick) {
|
||||
annotate({
|
||||
action: { name: "documents.summary.autoSkipped" },
|
||||
meta: { documentId, reason: "no-provider" },
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
// Re-assert egress consent for the picked provider. A local pick is ungated;
|
||||
// an external pick is authorised by the `documentsAutoAiRead` opt-in checked
|
||||
// above (the toggle short-circuits the gate). Belt-and-braces: a consent race
|
||||
// (opt-out flipped mid-flight) resolves to a no-op, never an egress.
|
||||
try {
|
||||
await assertDocumentEgressConsent({
|
||||
userId,
|
||||
providerType: pick.providerType,
|
||||
surface: "insights",
|
||||
});
|
||||
} catch (err) {
|
||||
annotate({
|
||||
action: { name: "documents.summary.autoSkipped" },
|
||||
meta: {
|
||||
documentId,
|
||||
reason:
|
||||
err instanceof ConsentRequiredError ? "consent" : "consent-error",
|
||||
},
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
const document = await loadOwnedDocument(userId, documentId);
|
||||
if (!document) return;
|
||||
|
||||
const vision = await prepareVisionInput(document, pick.pdfSupported);
|
||||
if (!vision.ok) {
|
||||
// Cannot read this file with the picked provider (decrypt miss, unsupported
|
||||
// type, or a PDF needing an Anthropic provider). Graceful no-op.
|
||||
annotate({
|
||||
action: { name: "documents.summary.autoSkipped" },
|
||||
meta: { documentId, reason: vision.reason },
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
const dateKey = buildDateKey();
|
||||
const reservation = await reserveBudget(
|
||||
userId,
|
||||
AI_BUDGETS.documentSummary.maxTokens,
|
||||
dateKey,
|
||||
resolveDailyCap([{ providerType: pick.entry.providerType }]),
|
||||
);
|
||||
// Budget exhausted → skip (no local fallback for a summary).
|
||||
if (!reservation.allowed) {
|
||||
annotate({
|
||||
action: { name: "documents.summary.autoSkipped" },
|
||||
meta: { documentId, reason: "budget" },
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
const { summary } = await runDocumentSummary({
|
||||
provider: pick.entry.instance,
|
||||
providerType: pick.providerType,
|
||||
images: vision.images,
|
||||
documents: vision.documents,
|
||||
});
|
||||
await reconcileSpend(
|
||||
userId,
|
||||
reservation.reserved,
|
||||
reservation.reserved,
|
||||
dateKey,
|
||||
);
|
||||
// Persist ENCRYPTED. `updateMany` scoped to `summaryEncrypted: null` so a
|
||||
// racing writer (a re-enqueue) never overwrites an existing summary — the
|
||||
// first write wins and a second run is a no-op.
|
||||
const written = await prisma.inboundDocument.updateMany({
|
||||
where: {
|
||||
id: documentId,
|
||||
userId,
|
||||
deletedAt: null,
|
||||
summaryEncrypted: null,
|
||||
},
|
||||
data: {
|
||||
summaryEncrypted: encryptDocumentSummary(summary),
|
||||
summaryGeneratedAt: new Date(),
|
||||
},
|
||||
});
|
||||
annotate({
|
||||
action: { name: "documents.summary.autoGenerated" },
|
||||
meta: {
|
||||
documentId,
|
||||
providerType: pick.providerType,
|
||||
length: summary.length,
|
||||
persisted: written.count > 0,
|
||||
},
|
||||
});
|
||||
} catch {
|
||||
// Refund the reservation on a provider miss; the document keeps no summary
|
||||
// (the on-demand route remains the manual fallback). Never rethrow — a
|
||||
// transient provider error must not retry-loop or fail the queue.
|
||||
await reconcileSpend(userId, reservation.reserved, 0, dateKey);
|
||||
annotate({
|
||||
action: { name: "documents.summary.autoFailed" },
|
||||
meta: { documentId, reason: "provider_error" },
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Enqueue a per-document summary job. Coalesced by `singletonKey` per document
|
||||
* so a duplicate/idempotent re-upload cannot pile up parallel runs. Fire-and-
|
||||
* forget — a missing boss (worker not up) is a no-op, and a `boss.send` failure
|
||||
* (a transient DB hiccup) is swallowed to a no-op too, so enqueue can NEVER fail
|
||||
* an already-stored upload. The summary is a background nicety; the stored
|
||||
* document is the contract.
|
||||
*/
|
||||
export async function enqueueDocumentSummary(
|
||||
userId: string,
|
||||
documentId: string,
|
||||
): Promise<{ enqueued: boolean }> {
|
||||
const boss = getGlobalBoss();
|
||||
if (!boss) return { enqueued: false };
|
||||
const payload: DocumentSummaryPayload = {
|
||||
userId,
|
||||
documentId,
|
||||
enqueuedAt: new Date().toISOString(),
|
||||
};
|
||||
try {
|
||||
const jobId = await boss.send(DOCUMENT_SUMMARY_QUEUE, payload, {
|
||||
retryLimit: 2,
|
||||
retryDelay: 30,
|
||||
retryBackoff: true,
|
||||
singletonKey: `document-summary|${documentId}`,
|
||||
});
|
||||
return { enqueued: Boolean(jobId) };
|
||||
} catch (err) {
|
||||
annotate({
|
||||
action: { name: "documents.summary.enqueueFailed" },
|
||||
meta: { documentId, reason: err instanceof Error ? err.name : "unknown" },
|
||||
});
|
||||
return { enqueued: false };
|
||||
}
|
||||
}
|
||||
@@ -87,6 +87,12 @@ import {
|
||||
runDocumentThumbnail,
|
||||
type DocumentThumbnailPayload,
|
||||
} from "@/lib/jobs/document-thumbnail";
|
||||
import {
|
||||
DOCUMENT_SUMMARY_QUEUE,
|
||||
DOCUMENT_SUMMARY_CONCURRENCY,
|
||||
runDocumentSummaryJob,
|
||||
type DocumentSummaryPayload,
|
||||
} from "@/lib/jobs/document-summary";
|
||||
import {
|
||||
DOCUMENT_THUMBNAIL_BACKFILL_QUEUE,
|
||||
DOCUMENT_THUMBNAIL_BACKFILL_CONCURRENCY,
|
||||
@@ -344,6 +350,11 @@ const allQueues = [
|
||||
// thumbnail jobs. Without this entry pg-boss never provisions the queue and
|
||||
// the boot discovery silently no-ops.
|
||||
DOCUMENT_THUMBNAIL_BACKFILL_QUEUE,
|
||||
// Document AI — automatic per-document plain-language summary, enqueued on
|
||||
// upload. Provider (vision) only, gated on the `documentsAutoAiRead` opt-in +
|
||||
// egress consent + budget; persists the summary encrypted. Without this entry
|
||||
// pg-boss never provisions the queue and every upload enqueue silently no-ops.
|
||||
DOCUMENT_SUMMARY_QUEUE,
|
||||
];
|
||||
|
||||
const schedules: ScheduleEntry[] = [
|
||||
@@ -768,6 +779,33 @@ export async function registerMaintenanceQueues(
|
||||
},
|
||||
);
|
||||
|
||||
// Document AI — automatic per-document plain-language summary. Enqueued on
|
||||
// upload (one job per stored document); runs the vision provider ONLY when
|
||||
// the `documentsAutoAiRead` opt-in is ON (egress consent + budget gated) and
|
||||
// persists the summary encrypted. Serial concurrency so the provider call
|
||||
// never crowds the request pool.
|
||||
await boss.work<DocumentSummaryPayload>(
|
||||
DOCUMENT_SUMMARY_QUEUE,
|
||||
{ localConcurrency: DOCUMENT_SUMMARY_CONCURRENCY },
|
||||
async (jobs) => {
|
||||
for (const job of jobs) {
|
||||
const { userId, documentId } = job.data;
|
||||
if (!userId || !documentId) continue;
|
||||
try {
|
||||
await runDocumentSummaryJob(job.data);
|
||||
} catch (err) {
|
||||
recordError();
|
||||
workerLog(
|
||||
"error",
|
||||
`[document-summary] user=${userId} document=${documentId} failed`,
|
||||
err,
|
||||
);
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
// Document vault — boot-time preview-thumbnail backfill worker. The boot
|
||||
// discovery sends one per-user job; this handler fans out per-document
|
||||
// thumbnail jobs for that user's not-yet-thumbnailed documents. Serial
|
||||
|
||||
@@ -161,8 +161,16 @@ const inboundDocument = z
|
||||
});
|
||||
|
||||
const inboundDocumentDetail = inboundDocument
|
||||
.extend({ facts: z.array(extractedFact) })
|
||||
.meta({ id: "InboundDocumentDetail" });
|
||||
.extend({
|
||||
facts: z.array(extractedFact),
|
||||
summary: z.string().nullable(),
|
||||
summaryGeneratedAt: z.string().nullable(),
|
||||
})
|
||||
.meta({
|
||||
id: "InboundDocumentDetail",
|
||||
description:
|
||||
"A stored document plus its staged facts. `summary` is a short (3-4 sentence) plain-language description of WHAT the document is, generated once in the background right after upload when the `documentsAutoAiRead` opt-in is ON (that flag is also the standing AI-egress consent); it is descriptive only, never a diagnosis, and is null when auto-read is OFF, no provider is configured, or the background summary has not run yet. `summaryGeneratedAt` is when it was generated (null until then).",
|
||||
});
|
||||
|
||||
const listResponse = z
|
||||
.object({
|
||||
|
||||
@@ -292,6 +292,15 @@ export interface InboundDocumentDto {
|
||||
|
||||
export interface InboundDocumentDetailDto extends InboundDocumentDto {
|
||||
facts: ExtractedFactDto[];
|
||||
/**
|
||||
* Short (3-4 sentence) plain-language summary of WHAT the document is,
|
||||
* generated once in the background after upload when the `documentsAutoAiRead`
|
||||
* opt-in is ON. Null when auto-read is OFF, no provider is configured, or the
|
||||
* background summary has not run yet. Descriptive only — never a diagnosis.
|
||||
*/
|
||||
summary: string | null;
|
||||
/** When the background summary was generated (ISO 8601), or null. */
|
||||
summaryGeneratedAt: string | null;
|
||||
}
|
||||
|
||||
// ─── Edit (correction before approval) ─────────────────────────────────────
|
||||
|
||||
Reference in new issue
Block a user