From 5a0fd47eb50875e2d66d295097e816597f36384f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Tue, 7 Jul 2026 18:21:51 +0200 Subject: [PATCH] feat(documents): content-index backfill job, rotation coverage, OpenAPI MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit P2-1-T7/T8/T9. - document-content-index-backfill: on-demand per-user pg-boss job that indexes a user's not-yet-indexed image/PDF documents via one provider transcription each. Consent + budget gated, id-cursor forward walk, capped per run, stops on budget exhaustion, resumable. Queue + worker registered in the maintenance registrar (no cron — deliberate action only). POST /reindex triggers it. - rotation: register DocumentContentIndex.textEncrypted and add a dedicated block that re-encrypts the text AND re-tokenises the blind index under the new active HKDF subkey in one update (P2-D7). - OpenAPI: suggest / summary / index / reindex paths; hasContentIndex on the document schema; assistAvailable + contentIndex on usage. Regenerated docs/api/openapi.yaml. --- docs/api/openapi.yaml | 352 +++++++++++++++++- scripts/rotate-encryption-key.ts | 60 +++ .../api/documents/inbound/reindex/route.ts | 70 ++++ src/lib/crypto/encrypted-columns.ts | 9 + .../jobs/document-content-index-backfill.ts | 194 ++++++++++ src/lib/jobs/reminder/register-maintenance.ts | 43 +++ src/lib/openapi/routes/documents.ts | 175 ++++++++- 7 files changed, 901 insertions(+), 2 deletions(-) create mode 100644 src/app/api/documents/inbound/reindex/route.ts create mode 100644 src/lib/jobs/document-content-index-backfill.ts diff --git a/docs/api/openapi.yaml b/docs/api/openapi.yaml index cf449faec..919efa494 100644 --- a/docs/api/openapi.yaml +++ b/docs/api/openapi.yaml @@ -9135,6 +9135,197 @@ paths: $ref: "#/components/schemas/ErrorEnvelope" "422": *a3 "429": *a4 + /api/documents/inbound/{id}/suggest: + post: + tags: + - Documents + summary: Suggest filing metadata (drafts only) + description: "Optional AI assist on an already-stored document. Runs ONE provider call over the stored original (VISION, + empty body) or browser-OCR'd text (TEXT, `application/json` `{ mode: \"text\", text }`, opt-in local OCR) and + returns a `{ title, kind, documentDate }` DRAFT for the edit form. AI-consent / rate / budget gated (shares the + 6/hour extract bucket). WRITES NOTHING — never stages facts, never flips status; the user reviews and saves. 422 + (`documents.inbound.providerUnsupported`) with no provider configured. Never interprets or diagnoses; the title + is a neutral filing label." + parameters: + - name: id + in: path + required: true + schema: + type: string + requestBody: + required: false + content: + application/json: + schema: + type: object + properties: + mode: + type: string + const: text + text: + type: string + kind: + type: string + enum: + - DOCTOR_REPORT + - DISCHARGE_LETTER + - LAB_RESULT + - IMAGING + - PRESCRIPTION + - REFERRAL + - INSURANCE + - VACCINATION + - OTHER + required: + - mode + - text + responses: + "200": + description: The filing-metadata suggestion (drafts). + content: + application/json: + schema: + $ref: "#/components/schemas/DocumentSuggestEnvelope" + "401": *a1 + "422": *a3 + "429": *a4 + /api/documents/inbound/{id}/summary: + post: + tags: + - Documents + summary: Summarise or transcribe a document (session-only) + description: 'On-demand, SESSION-ONLY description of a stored document. `?mode=summary` (default) returns a short + plain-language summary of WHAT the document is; `?mode=text` returns its raw transcribed text. The result is + transient — nothing is persisted except the AI-budget ledger + audit log; it never reaches coach memory, + snapshots, the structured stores, or the search index. The summary is descriptive only and never a diagnosis. + Same VISION (empty body) / TEXT (`application/json` `{ mode: "text", text }`, opt-in local OCR) dispatch as + extract. AI-consent / rate / budget gated. 422 with no provider.' + parameters: + - name: id + in: path + required: true + schema: + type: string + - name: mode + in: query + required: false + schema: + type: string + enum: + - summary + - text + default: summary + description: "`summary` (plain-language description) or `text` (raw)." + requestBody: + required: false + content: + application/json: + schema: + type: object + properties: + mode: + type: string + const: text + text: + type: string + kind: + type: string + enum: + - DOCTOR_REPORT + - DISCHARGE_LETTER + - LAB_RESULT + - IMAGING + - PRESCRIPTION + - REFERRAL + - INSURANCE + - VACCINATION + - OTHER + required: + - mode + - text + responses: + "200": + description: The session-only summary (`{ summary }`) or extracted text (`{ text }`). + content: + application/json: + schema: + $ref: "#/components/schemas/DocumentSummaryEnvelope" + "401": *a1 + "422": *a3 + "429": *a4 + /api/documents/inbound/{id}/index: + post: + tags: + - Documents + summary: Build / refresh the content-search index + description: "Populates or refreshes one document's content-search index so search matches INSIDE its body. VISION + (empty body) decrypts the stored original and runs one provider transcription (AI-consent / rate / budget + gated); TEXT (`application/json` `{ mode: \"text\", text }`, opt-in local OCR) indexes browser-OCR'd text with + no provider egress. Persists ONLY AES-256-GCM ciphertext of the text plus opaque HMAC token hashes — no + plaintext body, no plaintext token. Gated on the existing AI consent (no separate per-user toggle). Idempotent; + re-indexing overwrites in place." + parameters: + - name: id + in: path + required: true + schema: + type: string + requestBody: + required: false + content: + application/json: + schema: + type: object + properties: + mode: + type: string + const: text + text: + type: string + kind: + type: string + enum: + - DOCTOR_REPORT + - DISCHARGE_LETTER + - LAB_RESULT + - IMAGING + - PRESCRIPTION + - REFERRAL + - INSURANCE + - VACCINATION + - OTHER + required: + - mode + - text + responses: + "200": + description: The document was indexed. + content: + application/json: + schema: + $ref: "#/components/schemas/DocumentIndexEnvelope" + "401": *a1 + "422": *a3 + "429": *a4 + /api/documents/inbound/reindex: + post: + tags: + - Documents + summary: Index all documents for content search + description: Enqueues a background job that content-indexes the caller's not-yet-indexed documents (one provider + transcription each, bounded + resumable). Gated on the module, a configured vision provider (422 + `documents.inbound.providerUnsupported` otherwise), and the existing AI consent (403 otherwise). Returns + immediately; the work runs off-request on the queue. + responses: + "200": + description: Whether a backfill job was enqueued. + content: + application/json: + schema: + $ref: "#/components/schemas/DocumentReindexEnvelope" + "401": *a1 + "422": *a3 + "429": *a4 /api/documents/inbound/{id}/facts/{factId}: patch: tags: @@ -25933,6 +26124,8 @@ components: enum: - inline - attachment + hasContentIndex: + type: boolean createdAt: type: string updatedAt: @@ -25953,6 +26146,7 @@ components: - pendingCount - conditionLinks - servingClass + - hasContentIndex - createdAt - updatedAt additionalProperties: false @@ -25960,7 +26154,8 @@ components: staging summary. `status` is STORED for a freshly uploaded file (no extraction run). `title` is the user label (plaintext); `documentDate` is the user filing date; `reportDate` is the model-transcribed date (null until extraction runs). `servingClass` says how `/original` delivers the file — render inline (`inline`) or - download-only (`attachment`); render/download by it, never by MIME guess. Treat an unknown `kind` value as OTHER + download-only (`attachment`); render/download by it, never by MIME guess. `hasContentIndex` is true when the + document has a content-search index (drives the re-index affordance). Treat an unknown `kind` value as OTHER when decoding. DocumentConditionLink: type: object @@ -26034,12 +26229,30 @@ components: - episodeId - name additionalProperties: false + assistAvailable: + type: boolean + contentIndex: + type: object + properties: + enabled: + type: boolean + indexedCount: + type: number + totalCount: + type: number + required: + - enabled + - indexedCount + - totalCount + additionalProperties: false required: - usedBytes - quotaBytes - maxFileBytes - acceptedExtensions - linkedEpisodes + - assistAvailable + - contentIndex additionalProperties: false DocumentBulkEnvelope: type: object @@ -26166,6 +26379,8 @@ components: enum: - inline - attachment + hasContentIndex: + type: boolean createdAt: type: string updatedAt: @@ -26190,6 +26405,7 @@ components: - pendingCount - conditionLinks - servingClass + - hasContentIndex - createdAt - updatedAt - facts @@ -26336,6 +26552,140 @@ components: - data - error additionalProperties: false + DocumentSuggestEnvelope: + type: object + properties: + data: + $ref: "#/components/schemas/DocumentSuggestResponse" + error: + type: "null" + meta: + type: object + properties: + requestId: + type: string + additionalProperties: false + required: + - data + - error + additionalProperties: false + DocumentSuggestResponse: + type: object + properties: + suggestions: + type: object + properties: + title: + anyOf: + - type: string + - type: "null" + kind: + anyOf: + - type: string + enum: + - DOCTOR_REPORT + - DISCHARGE_LETTER + - LAB_RESULT + - IMAGING + - PRESCRIPTION + - REFERRAL + - INSURANCE + - VACCINATION + - OTHER + - type: "null" + documentDate: + anyOf: + - type: string + - type: "null" + required: + - title + - kind + - documentDate + additionalProperties: false + required: + - suggestions + additionalProperties: false + DocumentSummaryEnvelope: + type: object + properties: + data: + $ref: "#/components/schemas/DocumentSummaryResponse" + error: + type: "null" + meta: + type: object + properties: + requestId: + type: string + additionalProperties: false + required: + - data + - error + additionalProperties: false + DocumentSummaryResponse: + type: object + properties: + summary: + type: string + text: + type: string + additionalProperties: false + DocumentIndexEnvelope: + type: object + properties: + data: + $ref: "#/components/schemas/DocumentIndexResponse" + error: + type: "null" + meta: + type: object + properties: + requestId: + type: string + additionalProperties: false + required: + - data + - error + additionalProperties: false + DocumentIndexResponse: + type: object + properties: + documentId: + type: string + indexed: + type: boolean + tokenCount: + type: number + required: + - documentId + - indexed + - tokenCount + additionalProperties: false + DocumentReindexEnvelope: + type: object + properties: + data: + $ref: "#/components/schemas/DocumentReindexResponse" + error: + type: "null" + meta: + type: object + properties: + requestId: + type: string + additionalProperties: false + required: + - data + - error + additionalProperties: false + DocumentReindexResponse: + type: object + properties: + enqueued: + type: boolean + required: + - enqueued + additionalProperties: false ExtractedFactEnvelope: type: object properties: diff --git a/scripts/rotate-encryption-key.ts b/scripts/rotate-encryption-key.ts index ebad6121a..525bbf03d 100644 --- a/scripts/rotate-encryption-key.ts +++ b/scripts/rotate-encryption-key.ts @@ -28,6 +28,7 @@ import { rotateColumn, type CorpusClient, } from "@/lib/crypto/encryption-corpus"; +import { tokeniseAndHash } from "@/lib/documents/content-index"; const DATABASE_URL = process.env.DATABASE_URL; if (!DATABASE_URL) { @@ -530,6 +531,65 @@ async function main() { ), ); + // ───── v1.27.22 document content-search index (Bytes text + re-tokenise) ───── + // The blind content index carries TWO coupled artefacts under the index key + // story: `text_encrypted` (the `encrypt()`-string-as-UTF-8 Bytes shape) AND + // `search_tokens` (HMAC-SHA256 under an HKDF subkey derived from the ACTIVE + // key). Rotating the master key changes that subkey, so a plain Bytes rotation + // of the text alone would leave the tokens hashed under the OLD subkey and + // search would silently miss the row. This dedicated block re-encrypts the + // text AND re-tokenises from the decrypted plaintext under the NEW subkey in + // the same update (P2-D7). Bounded id-cursor batches — the text is capped but + // still a blob, so an unbounded findMany is avoided. Idempotent: rows already + // on the active key are skipped, so an interrupted run resumes safely. + { + const result: RotationResult = { + table: "DocumentContentIndex", + field: "textEncrypted", + scanned: 0, + rotated: 0, + errors: 0, + }; + let cursor: string | null = null; + for (;;) { + const rows = await prisma.documentContentIndex.findMany({ + select: { id: true, textEncrypted: true }, + orderBy: { id: "asc" }, + take: 100, + ...(cursor ? { cursor: { id: cursor }, skip: 1 } : {}), + }); + if (rows.length === 0) break; + for (const row of rows) { + result.scanned++; + const buf = row.textEncrypted as Uint8Array | null; + if (!buf || buf.byteLength === 0) continue; + const asString = Buffer.from(buf).toString("utf8"); + if (!shouldRotate(asString)) continue; + try { + const plaintext = decrypt(asString); + const reEnc = encrypt(plaintext); + const encoded = Buffer.from(reEnc, "utf8"); + const nextBytes = new Uint8Array(new ArrayBuffer(encoded.byteLength)); + nextBytes.set(encoded); + const searchTokens = tokeniseAndHash(plaintext); + await prisma.documentContentIndex.update({ + where: { id: row.id }, + data: { textEncrypted: nextBytes, searchTokens }, + }); + result.rotated++; + } catch (err) { + result.errors++; + console.error( + `[DocumentContentIndex.textEncrypted] row ${row.id}: ${(err as Error).message}`, + ); + } + } + cursor = rows[rows.length - 1]!.id; + if (rows.length < 100) break; + } + results.push(result); + } + console.log("\n=== Rotation summary ==="); let totalRotated = 0; let totalErrors = 0; diff --git a/src/app/api/documents/inbound/reindex/route.ts b/src/app/api/documents/inbound/reindex/route.ts new file mode 100644 index 000000000..0b396e09c --- /dev/null +++ b/src/app/api/documents/inbound/reindex/route.ts @@ -0,0 +1,70 @@ +/** + * v1.27.22 (Document vault P2) — trigger the content-search index backfill. + * + * Enqueues a per-user job that indexes the caller's not-yet-indexed documents + * (one provider transcription each, bounded + resumable). Gated on the module, + * a configured vision provider, and the EXISTING AI consent (the worker + * re-checks all three). The work runs off-request on pg-boss; this route only + * enqueues and returns immediately so the vault stays responsive. + */ +import { apiHandler, requireAuth } from "@/lib/api-handler"; +import { apiError, apiSuccess, getClientIp } from "@/lib/api-response"; +import { assertConsentForChain } from "@/lib/ai/consent-guard"; +import { auditLog } from "@/lib/auth/audit"; +import { enqueueContentIndexBackfill } from "@/lib/jobs/document-content-index-backfill"; +import { resolveVisionProvider } from "@/lib/labs/ocr-capability"; +import { annotate } from "@/lib/logging/context"; +import { requireModuleEnabled } from "@/lib/modules/gate"; +import { checkRateLimit, rateLimitHeaders } from "@/lib/rate-limit"; + +export const dynamic = "force-dynamic"; + +/** A generous per-user ceiling — enqueue is cheap, the worker is the real gate. */ +const REINDEX_LIMIT_PER_HOUR = 12; +const REINDEX_WINDOW_MS = 60 * 60 * 1000; + +export const POST = apiHandler(async (request) => { + const { user } = await requireAuth(); + + const gate = await requireModuleEnabled(user.id, "inboundDocuments"); + if (!gate.enabled) return gate.response; + + const rl = await checkRateLimit( + `documents-reindex:${user.id}`, + REINDEX_LIMIT_PER_HOUR, + REINDEX_WINDOW_MS, + ); + if (!rl.allowed) { + const response = apiError("Too many requests. Try again later.", 429, { + errorCode: "documents.inbound.rateLimited", + }); + for (const [k, v] of Object.entries(rateLimitHeaders(rl))) { + response.headers.set(k, v); + } + return response; + } + + // Fail fast when the precondition is not met so the UI gets immediate + // feedback rather than a silently no-op'd job. + const { chain, pick } = await resolveVisionProvider(user.id); + if (!pick) { + return apiError("No vision-capable AI provider is configured", 422, { + errorCode: "documents.inbound.providerUnsupported", + }); + } + await assertConsentForChain({ userId: user.id, chain, surface: "insights" }); + + const { enqueued } = await enqueueContentIndexBackfill(user.id); + + await auditLog("documents.inbound.reindex", { + userId: user.id, + ipAddress: getClientIp(request), + details: { enqueued }, + }); + annotate({ + action: { name: "documents.contentIndex.backfillEnqueue" }, + meta: { enqueued }, + }); + + return apiSuccess({ enqueued }); +}); diff --git a/src/lib/crypto/encrypted-columns.ts b/src/lib/crypto/encrypted-columns.ts index 1bc6d6760..f1d84abf8 100644 --- a/src/lib/crypto/encrypted-columns.ts +++ b/src/lib/crypto/encrypted-columns.ts @@ -208,6 +208,15 @@ export const ENCRYPTED_COLUMNS: readonly EncryptedColumn[] = [ // document itself rather than persisting as plaintext JSONB. { model: "ExtractedFact", field: "dataEncrypted", kind: "bytes" }, { model: "ExtractedFact", field: "provenanceEncrypted", kind: "bytes" }, + + // ───── v1.27.22 document content-search index (Bytes text) ───── + // The normalised extracted text of an indexed document, AES-256-GCM at rest + // in the `encrypt()`-string-as-UTF-8 Bytes shape. Rotation re-encrypts it AND + // re-tokenises the sibling blind token array from the decrypted text under the + // new active index subkey (P2-D7) — a dedicated block in the rotation script, + // not the generic Bytes walk. The `search_tokens` array itself is one-way + // (HMAC) and is not a registry column. + { model: "DocumentContentIndex", field: "textEncrypted", kind: "bytes" }, ] as const; /** Stable `Model.field` key for a registry entry. */ diff --git a/src/lib/jobs/document-content-index-backfill.ts b/src/lib/jobs/document-content-index-backfill.ts new file mode 100644 index 000000000..729144753 --- /dev/null +++ b/src/lib/jobs/document-content-index-backfill.ts @@ -0,0 +1,194 @@ +/** + * v1.27.22 (Document vault P2) — content-search index backfill. + * + * Indexes a user's EXISTING stored documents for content search: for each live + * document that has no `DocumentContentIndex` yet, run one provider vision + * transcription and upsert the encrypted-text + blind-token index. On-demand + * only (fired by the "index all documents" action) — NOT a boot/cron pass: + * indexing egresses to a provider under the user's key + budget, so it must be + * a deliberate, consented act, exactly like the per-document index route. + * + * Consent: gated on the EXISTING AI consent (`assertConsentForChain`), the same + * gate the extract / index routes use — there is no separate per-user toggle + * (maintainer decision, 2026-07-07). + * + * Bounded + resumable: an id-cursor forward walk over the not-yet-indexed set, + * capped at `MAX_DOCS_PER_RUN` provider calls per job and stopped the moment the + * daily AI budget is reached. Only vision-indexable MIME types (image + PDF when + * the provider reads PDFs) are candidates, so the walk converges — an + * un-indexable file never re-queues work. A re-run picks up where budget / + * the cap left off because already-indexed documents drop out of the set. + * + * The queue MUST be registered in the maintenance registrar + * (`src/lib/jobs/reminder/register-maintenance.ts`) so pg-boss provisions it. + */ +import { AI_BUDGETS } from "@/lib/ai/ai-budgets"; +import { assertConsentForChain, ConsentRequiredError } from "@/lib/ai/consent-guard"; +import { + buildDateKey, + reconcileSpend, + reserveBudget, + resolveDailyCap, +} from "@/lib/ai/coach/budget"; +import { prisma } from "@/lib/db"; +import { + loadOwnedDocument, + prepareVisionInput, +} from "@/lib/documents/ai-route-support"; +import { upsertContentIndex } from "@/lib/documents/content-index"; +import { transcribeDocument } from "@/lib/documents/describe"; +import { getGlobalBoss } from "@/lib/jobs/boss-instance"; +import { resolveVisionProvider } from "@/lib/labs/ocr-capability"; +import { annotate } from "@/lib/logging/context"; + +export const CONTENT_INDEX_BACKFILL_QUEUE = "document-content-index-backfill"; + +/** Serial concurrency — provider calls, kept off the request pool. */ +export const CONTENT_INDEX_BACKFILL_CONCURRENCY = 1; + +/** Discovery page size (id-only; blobs are loaded one at a time). */ +const PAGE_SIZE = 50; + +/** Max documents one job indexes before yielding (bounds spend + runtime). */ +const MAX_DOCS_PER_RUN = 200; + +/** Image MIME types the vision path can always index. */ +const IMAGE_MIMES = ["image/jpeg", "image/png", "image/webp"] as const; + +export interface ContentIndexBackfillPayload { + userId: string; + enqueuedAt?: string; +} + +export interface ContentIndexBackfillSummary { + indexed: number; + reason: "ok" | "no-provider" | "no-consent" | "budget-reached"; +} + +/** + * Index one user's not-yet-indexed documents. Idempotent + resumable: an + * already-indexed document drops out of the candidate set, and the run stops at + * the per-run cap or when the budget is reached. + */ +export async function runContentIndexBackfillForUser( + userId: string, +): Promise { + const { chain, pick } = await resolveVisionProvider(userId); + if (!pick) return { indexed: 0, reason: "no-provider" }; + try { + await assertConsentForChain({ userId, chain, surface: "insights" }); + } catch (err) { + if (err instanceof ConsentRequiredError) { + return { indexed: 0, reason: "no-consent" }; + } + throw err; + } + + const candidateMimes = pick.pdfSupported + ? [...IMAGE_MIMES, "application/pdf"] + : [...IMAGE_MIMES]; + + const dailyCap = resolveDailyCap([{ providerType: pick.entry.providerType }]); + let indexed = 0; + let reason: ContentIndexBackfillSummary["reason"] = "ok"; + let cursor: string | null = null; + + outer: for (;;) { + if (indexed >= MAX_DOCS_PER_RUN) break; + const rows: { id: string }[] = await prisma.inboundDocument.findMany({ + where: { + userId, + deletedAt: null, + mimeType: { in: candidateMimes }, + contentIndex: { is: null }, + }, + select: { id: true }, + orderBy: { id: "asc" }, + take: PAGE_SIZE, + ...(cursor ? { cursor: { id: cursor }, skip: 1 } : {}), + }); + if (rows.length === 0) break; + + for (const { id } of rows) { + if (indexed >= MAX_DOCS_PER_RUN) break outer; + + const dateKey = buildDateKey(); + const reservation = await reserveBudget( + userId, + AI_BUDGETS.documentTranscribe.maxTokens, + dateKey, + dailyCap, + ); + if (!reservation.allowed) { + reason = "budget-reached"; + break outer; + } + + const document = await loadOwnedDocument(userId, id); + if (!document) { + await reconcileSpend(userId, reservation.reserved, 0, dateKey); + continue; + } + const vision = prepareVisionInput(document, pick.pdfSupported); + if (!vision.ok) { + // Not vision-indexable (should be filtered by MIME, but guard) — refund + // and move the cursor past it so the walk keeps converging. + await reconcileSpend(userId, reservation.reserved, 0, dateKey); + continue; + } + + try { + const { text } = await transcribeDocument({ + provider: pick.entry.instance, + providerType: pick.providerType, + images: vision.images, + documents: vision.documents, + }); + await reconcileSpend(userId, reservation.reserved, reservation.reserved, dateKey); + await upsertContentIndex({ + userId, + documentId: document.id, + text, + source: "vision", + providerType: pick.providerType, + }); + indexed += 1; + } catch { + // A provider miss on one document must not abort the batch — refund and + // leave it un-indexed (a later run retries it). + await reconcileSpend(userId, reservation.reserved, 0, dateKey); + } + } + + cursor = rows[rows.length - 1]!.id; + if (rows.length < PAGE_SIZE) break; + } + + annotate({ + action: { name: "documents.contentIndex.backfill" }, + meta: { indexed, reason }, + }); + return { indexed, reason }; +} + +/** + * Enqueue a per-user content-index backfill. Coalesced by `singletonKey` so a + * user cannot pile up parallel runs; returns whether a job was created. + */ +export async function enqueueContentIndexBackfill( + userId: string, +): Promise<{ enqueued: boolean }> { + const boss = getGlobalBoss(); + if (!boss) return { enqueued: false }; + const payload: ContentIndexBackfillPayload = { + userId, + enqueuedAt: new Date().toISOString(), + }; + const jobId = await boss.send(CONTENT_INDEX_BACKFILL_QUEUE, payload, { + retryLimit: 3, + retryDelay: 60, + retryBackoff: true, + singletonKey: `document-content-index-backfill|${userId}`, + }); + return { enqueued: Boolean(jobId) }; +} diff --git a/src/lib/jobs/reminder/register-maintenance.ts b/src/lib/jobs/reminder/register-maintenance.ts index 507cb20c8..f44868e2b 100644 --- a/src/lib/jobs/reminder/register-maintenance.ts +++ b/src/lib/jobs/reminder/register-maintenance.ts @@ -69,6 +69,12 @@ import { enqueueBootTimeMedNotesEncryptionBackfill, type MedNotesEncryptionBackfillPayload, } from "@/lib/jobs/med-notes-encryption-backfill"; +import { + CONTENT_INDEX_BACKFILL_QUEUE, + CONTENT_INDEX_BACKFILL_CONCURRENCY, + runContentIndexBackfillForUser, + type ContentIndexBackfillPayload, +} from "@/lib/jobs/document-content-index-backfill"; import { ENCRYPTION_KEY_ROTATE_QUEUE, ENCRYPTION_KEY_ROTATE_CONCURRENCY, @@ -298,6 +304,12 @@ const allQueues = [ // the daily schedule silently no-ops and "deleted" documents hold backup // weight forever. DOCUMENT_PURGE_QUEUE, + // Document vault P2 — on-demand content-search index backfill. Fired by the + // "index all documents" action (no cron): indexes a user's not-yet-indexed + // documents via one provider transcription each, consent + budget gated. + // Without this entry pg-boss never provisions the queue and the trigger + // silently no-ops. + CONTENT_INDEX_BACKFILL_QUEUE, ]; const schedules: ScheduleEntry[] = [ @@ -639,6 +651,37 @@ export async function registerMaintenanceQueues( }, ); + // Document vault P2 — on-demand content-search index backfill worker. The + // trigger endpoint sends one per-user job; this handler indexes that user's + // not-yet-indexed documents (consent + budget gated, bounded + resumable). + // Serial concurrency so the provider calls never crowd the request pool. + await boss.work( + CONTENT_INDEX_BACKFILL_QUEUE, + { localConcurrency: CONTENT_INDEX_BACKFILL_CONCURRENCY }, + async (jobs) => { + for (const job of jobs) { + const { userId } = job.data; + if (!userId) continue; + try { + const { indexed, reason } = + await runContentIndexBackfillForUser(userId); + workerLog( + "info", + `[document-content-index-backfill] user=${userId} indexed=${indexed} reason=${reason}`, + ); + } catch (err) { + recordError(); + workerLog( + "error", + `[document-content-index-backfill] user=${userId} failed`, + err, + ); + throw err; + } + } + }, + ); + // v1.25 (W-ENV) — nightly environment fetch. The daily discovery tick (empty // payload) fans out one per-user job per opted-in account; the queue also // serves the on-demand backfill payloads from the settings surface. Serial diff --git a/src/lib/openapi/routes/documents.ts b/src/lib/openapi/routes/documents.ts index 4a0b50aa1..bc5a9b9cd 100644 --- a/src/lib/openapi/routes/documents.ts +++ b/src/lib/openapi/routes/documents.ts @@ -107,13 +107,14 @@ const inboundDocument = z pendingCount: z.number(), conditionLinks: z.array(conditionLink), servingClass: z.enum(["inline", "attachment"]), + hasContentIndex: z.boolean(), createdAt: z.string(), updatedAt: z.string(), }) .meta({ id: "InboundDocument", description: - "A stored document. The raw bytes are stored encrypted at rest and never returned; this is the metadata + staging summary. `status` is STORED for a freshly uploaded file (no extraction run). `title` is the user label (plaintext); `documentDate` is the user filing date; `reportDate` is the model-transcribed date (null until extraction runs). `servingClass` says how `/original` delivers the file — render inline (`inline`) or download-only (`attachment`); render/download by it, never by MIME guess. Treat an unknown `kind` value as OTHER when decoding.", + "A stored document. The raw bytes are stored encrypted at rest and never returned; this is the metadata + staging summary. `status` is STORED for a freshly uploaded file (no extraction run). `title` is the user label (plaintext); `documentDate` is the user filing date; `reportDate` is the model-transcribed date (null until extraction runs). `servingClass` says how `/original` delivers the file — render inline (`inline`) or download-only (`attachment`); render/download by it, never by MIME guess. `hasContentIndex` is true when the document has a content-search index (drives the re-index affordance). Treat an unknown `kind` value as OTHER when decoding.", }); const inboundDocumentDetail = inboundDocument @@ -354,6 +355,12 @@ export const inboundDocumentPaths: NonNullable = { name: z.string(), }), ), + assistAvailable: z.boolean(), + contentIndex: z.object({ + enabled: z.boolean(), + indexedCount: z.number(), + totalCount: z.number(), + }), }) .meta({ id: "DocumentUsage" }), "DocumentUsageEnvelope", @@ -609,6 +616,172 @@ export const inboundDocumentPaths: NonNullable = { }, }, }, + "/api/documents/inbound/{id}/suggest": { + post: { + tags: ["Documents"], + summary: "Suggest filing metadata (drafts only)", + description: + "Optional AI assist on an already-stored document. Runs ONE provider call over the stored original (VISION, empty body) or browser-OCR'd text (TEXT, `application/json` `{ mode: \"text\", text }`, opt-in local OCR) and returns a `{ title, kind, documentDate }` DRAFT for the edit form. AI-consent / rate / budget gated (shares the 6/hour extract bucket). WRITES NOTHING — never stages facts, never flips status; the user reviews and saves. 422 (`documents.inbound.providerUnsupported`) with no provider configured. Never interprets or diagnoses; the title is a neutral filing label.", + parameters: [ + { name: "id", in: "path", required: true, schema: { type: "string" } }, + ], + requestBody: { + required: false, + content: { + "application/json": { + schema: z.object({ + mode: z.literal("text"), + text: z.string(), + kind: kindEnum.optional(), + }), + }, + }, + }, + responses: { + "200": { + description: "The filing-metadata suggestion (drafts).", + content: { + "application/json": { + schema: dataEnvelope( + z + .object({ + suggestions: z.object({ + title: z.string().nullable(), + kind: kindEnum.nullable(), + documentDate: z.string().nullable(), + }), + }) + .meta({ id: "DocumentSuggestResponse" }), + "DocumentSuggestEnvelope", + ), + }, + }, + }, + ...stdResponses, + }, + }, + }, + "/api/documents/inbound/{id}/summary": { + post: { + tags: ["Documents"], + summary: "Summarise or transcribe a document (session-only)", + description: + "On-demand, SESSION-ONLY description of a stored document. `?mode=summary` (default) returns a short plain-language summary of WHAT the document is; `?mode=text` returns its raw transcribed text. The result is transient — nothing is persisted except the AI-budget ledger + audit log; it never reaches coach memory, snapshots, the structured stores, or the search index. The summary is descriptive only and never a diagnosis. Same VISION (empty body) / TEXT (`application/json` `{ mode: \"text\", text }`, opt-in local OCR) dispatch as extract. AI-consent / rate / budget gated. 422 with no provider.", + parameters: [ + { name: "id", in: "path", required: true, schema: { type: "string" } }, + { + name: "mode", + in: "query", + required: false, + schema: { + type: "string", + enum: ["summary", "text"], + default: "summary", + }, + description: "`summary` (plain-language description) or `text` (raw).", + }, + ], + requestBody: { + required: false, + content: { + "application/json": { + schema: z.object({ + mode: z.literal("text"), + text: z.string(), + kind: kindEnum.optional(), + }), + }, + }, + }, + responses: { + "200": { + description: + "The session-only summary (`{ summary }`) or extracted text (`{ text }`).", + content: { + "application/json": { + schema: dataEnvelope( + z + .object({ + summary: z.string().optional(), + text: z.string().optional(), + }) + .meta({ id: "DocumentSummaryResponse" }), + "DocumentSummaryEnvelope", + ), + }, + }, + }, + ...stdResponses, + }, + }, + }, + "/api/documents/inbound/{id}/index": { + post: { + tags: ["Documents"], + summary: "Build / refresh the content-search index", + description: + "Populates or refreshes one document's content-search index so search matches INSIDE its body. VISION (empty body) decrypts the stored original and runs one provider transcription (AI-consent / rate / budget gated); TEXT (`application/json` `{ mode: \"text\", text }`, opt-in local OCR) indexes browser-OCR'd text with no provider egress. Persists ONLY AES-256-GCM ciphertext of the text plus opaque HMAC token hashes — no plaintext body, no plaintext token. Gated on the existing AI consent (no separate per-user toggle). Idempotent; re-indexing overwrites in place.", + parameters: [ + { name: "id", in: "path", required: true, schema: { type: "string" } }, + ], + requestBody: { + required: false, + content: { + "application/json": { + schema: z.object({ + mode: z.literal("text"), + text: z.string(), + kind: kindEnum.optional(), + }), + }, + }, + }, + responses: { + "200": { + description: "The document was indexed.", + content: { + "application/json": { + schema: dataEnvelope( + z + .object({ + documentId: z.string(), + indexed: z.boolean(), + tokenCount: z.number(), + }) + .meta({ id: "DocumentIndexResponse" }), + "DocumentIndexEnvelope", + ), + }, + }, + }, + ...stdResponses, + }, + }, + }, + "/api/documents/inbound/reindex": { + post: { + tags: ["Documents"], + summary: "Index all documents for content search", + description: + "Enqueues a background job that content-indexes the caller's not-yet-indexed documents (one provider transcription each, bounded + resumable). Gated on the module, a configured vision provider (422 `documents.inbound.providerUnsupported` otherwise), and the existing AI consent (403 otherwise). Returns immediately; the work runs off-request on the queue.", + responses: { + "200": { + description: "Whether a backfill job was enqueued.", + content: { + "application/json": { + schema: dataEnvelope( + z + .object({ enqueued: z.boolean() }) + .meta({ id: "DocumentReindexResponse" }), + "DocumentReindexEnvelope", + ), + }, + }, + }, + ...stdResponses, + }, + }, + }, "/api/documents/inbound/{id}/facts/{factId}": { patch: { tags: ["Documents"],