From 91a5464aa3f3049f6225a627512f3d1b08286e8a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Fri, 23 Oct 2026 15:10:26 +0200 Subject: [PATCH] refactor(documents): move the vault upload reading and the list read out of their route --- .../baselines/source-file-lines.json | 5 +- .../baselines/source-function-lines.json | 5 +- .../baselines/version-stamp-comments.json | 4 +- src/app/api/documents/inbound/route.ts | 618 ++++-------------- src/lib/documents/upload-answer.ts | 66 ++ src/lib/documents/upload-request.ts | 142 ++++ src/lib/documents/upload-responses.ts | 144 ++++ src/lib/documents/vault-list.ts | 209 ++++++ 8 files changed, 689 insertions(+), 504 deletions(-) create mode 100644 src/lib/documents/upload-answer.ts create mode 100644 src/lib/documents/upload-request.ts create mode 100644 src/lib/documents/upload-responses.ts create mode 100644 src/lib/documents/vault-list.ts diff --git a/src/__tests__/baselines/source-file-lines.json b/src/__tests__/baselines/source-file-lines.json index e2c98387c..78c590442 100644 --- a/src/__tests__/baselines/source-file-lines.json +++ b/src/__tests__/baselines/source-file-lines.json @@ -1,10 +1,9 @@ { "$comment": "Production files over 800 lines and their length. Maintained by src/__tests__/source-size-ratchet.test.ts.", - "count": 104, - "total": 119957, + "count": 103, + "total": 119146, "entries": { "app/api/dashboard/summary/route.ts": 1032, - "app/api/documents/inbound/route.ts": 811, "app/api/insights/generate/route.ts": 1214, "app/medications/page-client.tsx": 839, "app/page-client.tsx": 2324, diff --git a/src/__tests__/baselines/source-function-lines.json b/src/__tests__/baselines/source-function-lines.json index d60990df5..feba73d69 100644 --- a/src/__tests__/baselines/source-function-lines.json +++ b/src/__tests__/baselines/source-function-lines.json @@ -1,7 +1,7 @@ { "$comment": "Production functions over 200 lines, keyed \"file :: name\", and their length. Maintained by src/__tests__/source-size-ratchet.test.ts.", - "count": 371, - "total": 129459, + "count": 370, + "total": 129150, "entries": { "app/achievements/page.tsx :: AchievementsPage": 206, "app/api/admin/backups/upload/route.ts :: apiHandler(arg)": 314, @@ -18,7 +18,6 @@ "app/api/dashboard/summary/route.ts :: buildDashboardSummary": 606, "app/api/devices/route.ts :: apiHandler(arg)": 231, "app/api/documents/inbound/bulk/route.ts :: apiHandler(arg)": 207, - "app/api/documents/inbound/route.ts :: processUpload": 309, "app/api/encounters/[id]/route.ts :: apiHandler(arg) #2": 269, "app/api/environment/route.ts :: apiHandler(arg)": 201, "app/api/export/health-record/route.ts :: apiHandler(arg)": 241, diff --git a/src/__tests__/baselines/version-stamp-comments.json b/src/__tests__/baselines/version-stamp-comments.json index c3f0cf3be..238eff99a 100644 --- a/src/__tests__/baselines/version-stamp-comments.json +++ b/src/__tests__/baselines/version-stamp-comments.json @@ -1,7 +1,7 @@ { "$comment": "Comment lines that open with a release version, per file under src/. Maintained by src/__tests__/version-stamp-ratchet.test.ts.", "count": 2911, - "total": 9689, + "total": 9688, "entries": { "__tests__/account-payload-consumer-guard.test.ts": 2, "__tests__/acting-account-boundary-guard.test.ts": 7, @@ -243,7 +243,7 @@ "app/api/documents/inbound/[id]/summary/route.ts": 2, "app/api/documents/inbound/[id]/thumbnail/route.ts": 1, "app/api/documents/inbound/reindex/route.ts": 1, - "app/api/documents/inbound/route.ts": 3, + "app/api/documents/inbound/route.ts": 2, "app/api/documents/inbound/usage/route.ts": 1, "app/api/environment/backfill/__tests__/route.test.ts": 1, "app/api/environment/route.ts": 1, diff --git a/src/app/api/documents/inbound/route.ts b/src/app/api/documents/inbound/route.ts index 86f68721c..2a8c35fb9 100644 --- a/src/app/api/documents/inbound/route.ts +++ b/src/app/api/documents/inbound/route.ts @@ -30,11 +30,13 @@ * * The document is UNTRUSTED (prompt-injection): the server never acts on an * instruction inside it. Storing it does nothing with its contents. + * + * The request reading, the stored-row answers and the list read live in + * `src/lib/documents/` (`upload-request.ts`, `upload-answer.ts`, + * `vault-list.ts`); the store itself is the shared `ingest.ts`. */ import { Buffer } from "node:buffer"; -import { NextResponse } from "next/server"; - import { apiHandler, isScopedCredential, @@ -50,48 +52,44 @@ import { sanitiseZodIssues, } from "@/lib/api-response"; import { getAiCapability } from "@/lib/ai/capabilities/gate"; -import { listPage } from "@/lib/http/list-query"; -import { prisma } from "@/lib/db"; -import { hashQueryTokens } from "@/lib/documents/content-index"; import { ingestDocument, - isoDateToUtc, personalUploadBucket, UPLOAD_LIMIT_PER_HOUR, UPLOAD_WINDOW_MS, } from "@/lib/documents/ingest"; -import { - loadConditionLinks, - loadDocumentEncounterLinks, -} from "@/lib/documents/links"; -import { - serialiseDocument, - type SerialisableDocument, -} from "@/lib/documents/store"; import { DOCUMENTS_WRITE_SCOPE } from "@/lib/documents/scopes"; import { checkSourceLookupRateLimit, findSourceKey, - type SourceKeyMatch, } from "@/lib/documents/source-key"; import { acquireDocumentUploadSlot, resolveDocumentLimits, resolveDocumentUploadLimitPerHour, } from "@/lib/documents/upload-policy"; +import { storedDocumentBody } from "@/lib/documents/upload-answer"; +import { + aliasLimitResponse, + answerSourceKey, + duplicateResponse, + fileTooLarge, + receiptResponse, + type UploadContext, +} from "@/lib/documents/upload-responses"; +import { + parseQuerySourceKey, + parseUploadMetadata, + readUploadForm, + resolveUploadSourceKey, + type QuerySourceKey, +} from "@/lib/documents/upload-request"; +import { listVaultDocuments } from "@/lib/documents/vault-list"; import { withIdempotency } from "@/lib/idempotency"; -import { BodyTooLargeError, readBoundedBody } from "@/lib/labs/ocr-upload"; -import { annotate } from "@/lib/logging/context"; import { requireModuleEnabled } from "@/lib/modules/gate"; import { checkRateLimit, rateLimitHeaders } from "@/lib/rate-limit"; -import { - documentCreateSchema, - documentListQuerySchema, - documentSourceKeySchema, - toContentIndexSource, - type DocumentSourceSystemValue, -} from "@/lib/validations/inbound-documents"; -import type { Prisma } from "@/generated/prisma/client"; +import type { IngestResult } from "@/lib/documents/ingest"; +import { documentListQuerySchema } from "@/lib/validations/inbound-documents"; export const dynamic = "force-dynamic"; @@ -101,201 +99,31 @@ export const dynamic = "force-dynamic"; * exact per-file cap is re-enforced on the extracted file bytes below. */ const MULTIPART_OVERHEAD_BYTES = 64 * 1024; -const UPLOAD_BODY_TIMEOUT_MS = 30_000; - -/** §3.2 — 413 fileTooLarge with the configured limit in `meta`. */ -function fileTooLarge(maxFileBytes: number): NextResponse { - return apiError("File is too large.", 413, { - errorCode: "documents.inbound.fileTooLarge", - reason: "fileTooLarge", - maxFileBytes, - }); -} - -/** What a single upload asked for, carried into the annotations. */ -interface UploadContext { - userId: string; - /** A narrow `documents:write` token, not a session or a `["*"]` token. */ - scoped: boolean; - sourceSystem: DocumentSourceSystemValue | null; - aiDeferred: boolean; -} /** - * The receipt a scoped caller gets in place of the stored row: the id and - * whether the upload was a duplicate. A write-only credential holder who - * knows a file's bytes learns that it is stored, not its title, filename, - * links or fact counts. + * A source key may ride the query string. It is answered here, before the + * bucket is charged and before a byte of the body is read: a nightly re-sync + * of an archive HealthLog already holds costs neither. Answers the response + * when the key settles the upload, else the key (or null) to carry on with. */ -function receiptResponse( - id: string | null, - status: 200 | 201, - flags: { duplicate: boolean; deleted?: boolean }, -): NextResponse { - const data = { - id, - duplicate: flags.duplicate, - ...(flags.deleted ? { deleted: true } : {}), - }; - return NextResponse.json( - { - data, - error: null, - ...(flags.duplicate - ? { - meta: { - duplicate: true, - ...(flags.deleted ? { deleted: true } : {}), - }, - } - : {}), - }, - { status }, - ); -} - -/** - * v1.39.2 (#1038) — a re-send of an imported document the person deleted. - * 200 and no new row, for every caller: the deletion was a decision, and an - * importer re-running over the same source must not undo it. `id` is the - * tombstoned row while it exists and null once the purge took it; there is no - * row left to serialise either way, so the answer is the receipt. - */ -function deletedResponse(ctx: UploadContext, id: string | null): NextResponse { - annotate({ - action: { name: "documents.vault.upload" }, - meta: { - documentId: id, - duplicate: true, - deleted: true, - scoped: ctx.scoped, - sourceSystem: ctx.sourceSystem, - aiDeferred: ctx.aiDeferred, - }, - }); - return receiptResponse(id, 200, { duplicate: true, deleted: true }); -} - -/** - * The same bytes have already been sent under too many source keys - * (`MAX_SOURCE_ALIASES_PER_DOCUMENT`). A conflict with what is stored, not a - * rate: nothing is remembered and nothing stored. - */ -function aliasLimitResponse(): NextResponse { - return apiError( - "This file is already stored under too many source ids.", - 409, - { errorCode: "documents.inbound.sourceAliasLimit" }, - ); -} - -/** Answer a source key that is already held: duplicate, or deleted. */ -function answerSourceKey( - ctx: UploadContext, - match: SourceKeyMatch, -): Promise | NextResponse { - return match.state === "live" - ? duplicateResponse(ctx, match.document) - : deletedResponse(ctx, match.id); -} - -/** - * §3.2 — a duplicate upload is NOT an error: return the existing live row - * with `meta.duplicate: true` at the envelope level (the UI toasts "already - * stored" and highlights the row). A scoped caller gets the receipt instead. - */ -async function duplicateResponse( - ctx: UploadContext, - existing: SerialisableDocument, -): Promise { - annotate({ - action: { name: "documents.vault.upload" }, - meta: { - documentId: existing.id, - duplicate: true, - scoped: ctx.scoped, - sourceSystem: ctx.sourceSystem, - aiDeferred: ctx.aiDeferred, - }, - }); - if (ctx.scoped) { - return receiptResponse(existing.id, 200, { duplicate: true }); - } - const userId = ctx.userId; - const [links, visitLinks, groups] = await Promise.all([ - loadConditionLinks(userId, [existing.id]), - loadDocumentEncounterLinks(userId, [existing.id]), - prisma.extractedFact.groupBy({ - by: ["status"], - where: { userId, documentId: existing.id }, - _count: { _all: true }, - }), - ]); - let factCount = 0; - let pendingCount = 0; - for (const g of groups) { - if (g.status !== "REJECTED") factCount += g._count._all; - if (g.status === "PENDING") pendingCount += g._count._all; - } - return NextResponse.json( - { - data: serialiseDocument( - existing, - { factCount, pendingCount }, - links.get(existing.id) ?? [], - false, - null, - false, - visitLinks.get(existing.id) ?? [], - ), - error: null, - meta: { duplicate: true }, - }, - { status: 200 }, - ); -} - -/** Process one admitted upload. The caller owns and releases its memory slot. */ -async function processUpload( +async function answerQuerySourceKey( request: Request, auth: AuthContext, -): Promise { + scoped: boolean, +): Promise { const { user } = auth; - const scoped = isScopedCredential(auth); - - // Opt-in module gate — even a valid Bearer token is refused when the surface - // is off (it ships dark; the user turns it on deliberately). - const gate = await requireModuleEnabled(user.id, "inboundDocuments"); - if (!gate.enabled) return gate.response; - - // A source key may ride the query string. It is answered here, before the - // bucket is charged and before a byte of the body is read: a nightly - // re-sync of an archive HealthLog already holds costs neither. Both halves - // or neither; half a key is a mistake worth saying so about. - const url = new URL(request.url); - const querySystem = url.searchParams.get("sourceSystem"); - const queryId = url.searchParams.get("sourceId"); - const queryInstance = url.searchParams.get("sourceInstance"); - let queryKey: { - sourceSystem: DocumentSourceSystemValue; - sourceId: string; - sourceInstance?: string; - } | null = null; - if (querySystem !== null || queryId !== null) { - const parsedKey = documentSourceKeySchema.safeParse({ - sourceSystem: querySystem ?? undefined, - sourceId: queryId ?? undefined, - sourceInstance: queryInstance ?? undefined, - }); - if (!parsedKey.success) { - return apiValidationError( - "Invalid source key", - sanitiseZodIssues(parsedKey.error.issues), - 422, - { errorCode: "documents.inbound.invalidMetadata" }, - ); - } - queryKey = parsedKey.data; + const query = parseQuerySourceKey(new URL(request.url)); + if (query.kind === "invalid") { + return apiValidationError( + "Invalid source key", + sanitiseZodIssues(query.issues), + 422, + { errorCode: "documents.inbound.invalidMetadata" }, + ); + } + const queryKey: QuerySourceKey | null = + query.kind === "key" ? query.key : null; + if (queryKey) { // Metered like the lookup route, on the same bucket: the check stores // nothing and costs no upload slot, but it is not a free oracle either. const lookupRl = await checkSourceLookupRateLimit( @@ -331,6 +159,71 @@ async function processUpload( } } + return queryKey; +} + +/** The answer to an upload the ingest did not store. */ +function answerNotStored( + ctx: UploadContext, + result: Exclude, +): Response | Promise { + const scoped = ctx.scoped; + switch (result.kind) { + case "sourceKey": + return answerSourceKey(ctx, result.match); + case "tooLarge": + return fileTooLarge(result.maxFileBytes); + case "empty": + return apiError("Uploaded file is empty", 422, { + errorCode: "documents.inbound.invalidMetadata", + }); + case "unsupportedType": + return apiError("This file type is not supported.", 415, { + errorCode: "documents.inbound.fileType", + reason: "unsupportedType", + }); + case "episodeNotFound": + return apiError("Episode not found", 404, { + errorCode: "documents.inbound.episodeNotFound", + }); + case "encounterNotFound": + return apiError("Visit not found", 404, { + errorCode: "documents.inbound.encounterNotFound", + }); + case "aliasLimit": + return aliasLimitResponse(); + case "duplicate": + return duplicateResponse(ctx, result.document); + case "quotaExceeded": + // The figures are the owner's; a write-only token learns only that the + // vault is full. + return apiError("Storage quota exceeded.", 413, { + errorCode: "documents.inbound.quotaExceeded", + reason: "quotaExceeded", + ...(scoped + ? {} + : { quotaBytes: result.quotaBytes, usedBytes: result.usedBytes }), + }); + } +} + +/** Process one admitted upload. The caller owns and releases its memory slot. */ +async function processUpload( + request: Request, + auth: AuthContext, +): Promise { + const { user } = auth; + const scoped = isScopedCredential(auth); + + // Opt-in module gate — even a valid Bearer token is refused when the surface + // is off (it ships dark; the user turns it on deliberately). + const gate = await requireModuleEnabled(user.id, "inboundDocuments"); + if (!gate.enabled) return gate.response; + + const query = await answerQuerySourceKey(request, auth, scoped); + if (query instanceof Response) return query; + const queryKey = query; + // A scoped token draws on its own bucket, keyed on the token (its id rides // `session.id` on the Bearer path), so an import running overnight cannot // use up the allowance the person's own uploads from the web or the phone @@ -367,55 +260,23 @@ async function processUpload( return fileTooLarge(limits.maxFileBytes); } - let formData: FormData; - try { - const bytes = await readBoundedBody(request.body, bodyCap, { - signal: request.signal, - deadline: Date.now() + UPLOAD_BODY_TIMEOUT_MS, + const read = await readUploadForm(request, bodyCap); + if (read.kind === "tooLarge") return fileTooLarge(limits.maxFileBytes); + if (read.kind === "timeout") { + return apiError("Upload body read timed out.", 408, { + errorCode: "documents.inbound.uploadTimeout", + reason: "uploadTimeout", }); - formData = await new Request(request.url, { - method: "POST", - headers: { "content-type": request.headers.get("content-type") ?? "" }, - body: bytes, - }).formData(); - } catch (err) { - if (err instanceof BodyTooLargeError) { - return fileTooLarge(limits.maxFileBytes); - } - if (err instanceof DOMException && err.name === "AbortError") { - return apiError("Upload body read timed out.", 408, { - errorCode: "documents.inbound.uploadTimeout", - reason: "uploadTimeout", - }); - } - return apiError("Invalid multipart body", 400); } + if (read.kind === "invalid") return apiError("Invalid multipart body", 400); + const formData = read.formData; const file = formData.get("file"); if (!(file instanceof File)) { return apiError("Field 'file' must be a file", 422); } - // Optional metadata (title / kind / documentDate / episodeIds / - // encounterIds). The file is read separately; these are the form fields - // beside it. Both id lists may be repeated. - const rawEpisodeIds = formData - .getAll("episodeIds") - .filter((v): v is string => typeof v === "string"); - const rawEncounterIds = formData - .getAll("encounterIds") - .filter((v): v is string => typeof v === "string"); - const parsed = documentCreateSchema.safeParse({ - title: formData.get("title") ?? undefined, - kind: formData.get("kind") ?? undefined, - documentDate: formData.get("documentDate") ?? undefined, - episodeIds: rawEpisodeIds.length > 0 ? rawEpisodeIds : undefined, - encounterIds: rawEncounterIds.length > 0 ? rawEncounterIds : undefined, - sourceSystem: formData.get("sourceSystem") ?? undefined, - sourceId: formData.get("sourceId") ?? undefined, - sourceInstance: formData.get("sourceInstance") ?? undefined, - aiRead: formData.get("aiRead") ?? undefined, - }); + const parsed = parseUploadMetadata(formData); if (!parsed.success) { return apiValidationError( "Invalid document metadata", @@ -427,30 +288,15 @@ async function processUpload( ); } - // The key from the query string, or from the form fields; if a caller sends - // both they have to agree. - const formSystem = parsed.data.sourceSystem ?? null; - const formId = formSystem ? (parsed.data.sourceId ?? null) : null; - const formInstance = formId ? (parsed.data.sourceInstance ?? null) : null; - if ( - queryKey && - formSystem !== null && - (formSystem !== queryKey.sourceSystem || - formId !== queryKey.sourceId || - (formInstance !== null && - formInstance !== (queryKey.sourceInstance ?? null))) - ) { + const key = resolveUploadSourceKey(queryKey, parsed.data); + if (key.kind === "mismatch") { return apiError( "The source key in the address and in the form differ.", 422, { errorCode: "documents.inbound.invalidMetadata" }, ); } - const sourceSystem = queryKey?.sourceSystem ?? formSystem; - const sourceId = queryKey?.sourceId ?? formId; - const sourceInstance = sourceId - ? (queryKey?.sourceInstance ?? formInstance ?? null) - : null; + const { sourceSystem, sourceId, sourceInstance } = key; const ctx: UploadContext = { userId: user.id, scoped, @@ -505,65 +351,11 @@ async function processUpload( documentAi: () => getAiCapability("documentAi"), }); - switch (result.kind) { - case "sourceKey": - return answerSourceKey(ctx, result.match); - case "tooLarge": - return fileTooLarge(result.maxFileBytes); - case "empty": - return apiError("Uploaded file is empty", 422, { - errorCode: "documents.inbound.invalidMetadata", - }); - case "unsupportedType": - return apiError("This file type is not supported.", 415, { - errorCode: "documents.inbound.fileType", - reason: "unsupportedType", - }); - case "episodeNotFound": - return apiError("Episode not found", 404, { - errorCode: "documents.inbound.episodeNotFound", - }); - case "encounterNotFound": - return apiError("Visit not found", 404, { - errorCode: "documents.inbound.encounterNotFound", - }); - case "aliasLimit": - return aliasLimitResponse(); - case "duplicate": - return duplicateResponse(ctx, result.document); - case "quotaExceeded": - // The figures are the owner's; a write-only token learns only that the - // vault is full. - return apiError("Storage quota exceeded.", 413, { - errorCode: "documents.inbound.quotaExceeded", - reason: "quotaExceeded", - ...(scoped - ? {} - : { quotaBytes: result.quotaBytes, usedBytes: result.usedBytes }), - }); - case "stored": - break; - } + if (result.kind !== "stored") return answerNotStored(ctx, result); const document = result.document; if (scoped) return receiptResponse(document.id, 201, { duplicate: false }); - - const [links, visitLinks] = await Promise.all([ - loadConditionLinks(user.id, [document.id]), - loadDocumentEncounterLinks(user.id, [document.id]), - ]); - return apiSuccess( - serialiseDocument( - document, - { factCount: 0, pendingCount: 0 }, - links.get(document.id) ?? [], - false, - null, - false, - visitLinks.get(document.id) ?? [], - ), - 201, - ); + return apiSuccess(await storedDocumentBody(user.id, document), 201); } /** Authenticate and reserve memory capacity before any request-body read. */ @@ -640,172 +432,6 @@ export const GET = apiHandler(async (request: Request) => { }, ); } - const { - q, - kind, - episodeId, - encounterId, - year, - from, - to, - sort, - order, - cursor, - limit, - } = parsed.data; - const where: Prisma.InboundDocumentWhereInput = { - userId: user.id, - deletedAt: null, - }; - if (kind && kind.length > 0) where.kind = { in: kind }; - if (episodeId) where.conditionLinks = { some: { episodeId } }; - // The visit filter reads the SAME table the visit's own sheet writes, from - // the document side. Filtering rather than post-filtering: a page of fifty - // documents narrowed in Node would page wrongly. - if (encounterId) where.encounterLinks = { some: { encounterId } }; - if (year !== undefined) { - where.documentDate = { - gte: new Date(Date.UTC(year, 0, 1)), - lt: new Date(Date.UTC(year + 1, 0, 1)), - }; - } else if (from || to) { - where.documentDate = { - ...(from ? { gte: isoDateToUtc(from) } : {}), - // inclusive end-of-day for the `to` bound - ...(to ? { lt: new Date(isoDateToUtc(to).getTime() + 86_400_000) } : {}), - }; - } - if (q) { - // Substring match on the short plaintext fields … - const or: Prisma.InboundDocumentWhereInput[] = [ - { title: { contains: q, mode: "insensitive" } }, - { filename: { contains: q, mode: "insensitive" } }, - ]; - // … unioned with a WHOLE-WORD content match over the blind token index. - // The query is tokenised + HMAC'd the same way the index was built, then - // matched with a GIN-accelerated array-overlap (`hasSome` → `&&`). The - // list still never selects the encrypted text — only the opaque hashes are - // touched, in the related table. Degrades silently to title/filename when - // the caller has no indexed documents (no rows overlap). - const hashes = hashQueryTokens(q); - if (hashes.length > 0) { - or.push({ contentIndex: { is: { searchTokens: { hasSome: hashes } } } }); - } - where.OR = or; - } - - // Keyset pagination on the sort column + a stable `id` tiebreak; nullable - // sort columns sort nulls last so undated documents trail. - const primaryOrder: - Prisma.InboundDocumentOrderByWithRelationInput | undefined = - sort === "documentDate" - ? { documentDate: { sort: order, nulls: "last" } } - : sort === "title" - ? { title: { sort: order, nulls: "last" } } - : { createdAt: order }; - const orderBy: Prisma.InboundDocumentOrderByWithRelationInput[] = [ - primaryOrder, - { id: order }, - ]; - - // Hardening: the list NEVER fetches the encrypted blob column — a page of - // 50 rows would otherwise drag up to 50 × cap ciphertext bytes per request. - const { rows: page, cap } = await listPage(limit, (take) => - prisma.inboundDocument.findMany({ - where, - omit: { contentEncrypted: true }, - orderBy, - take, - ...(cursor ? { cursor: { id: cursor }, skip: 1 } : {}), - }), - ); - - const hasMore = cap.truncated; - const nextCursor = hasMore ? (page[page.length - 1]?.id ?? null) : null; - - // Fact tallies for the whole page in ONE grouped count — no per-document - // fan-out and no materialising fact-id rows. `factCount` counts every - // non-REJECTED fact (a rejected fact is discarded, not part of the - // document's tally), `pendingCount` the PENDING subset awaiting review. - const factCounts = new Map< - string, - { factCount: number; pendingCount: number } - >(); - if (page.length > 0) { - const groups = await prisma.extractedFact.groupBy({ - by: ["documentId", "status"], - where: { userId: user.id, documentId: { in: page.map((d) => d.id) } }, - _count: { _all: true }, - }); - for (const g of groups) { - const entry = factCounts.get(g.documentId) ?? { - factCount: 0, - pendingCount: 0, - }; - const n = g._count._all; - if (g.status !== "REJECTED") entry.factCount += n; - if (g.status === "PENDING") entry.pendingCount += n; - factCounts.set(g.documentId, entry); - } - } - - // Condition/visit links + the content-index and thumbnail probes for the - // page, all in ONE Promise.all — four independent grouped queries (no - // N+1, never a ciphertext/blob column), and the vault polls this list - // every few seconds so two serialized round-trips were pure added - // latency (A5-4). - const pageIds = page.map((d) => d.id); - const [linkMap, visitLinkMap, indexedRows, thumbRows] = await Promise.all([ - loadConditionLinks(user.id, pageIds), - loadDocumentEncounterLinks(user.id, pageIds), - // Which of the page's documents have a content index (drives the - // searchable status + the provenance the UI reads to tell an AI-read - // document from a locally-indexed one). - page.length > 0 - ? prisma.documentContentIndex.findMany({ - where: { userId: user.id, documentId: { in: pageIds } }, - select: { documentId: true, source: true }, - }) - : Promise.resolve([]), - // Which of the page's documents have a preview thumbnail (gates the - // card's ). Grouped on the 1:1 side table. - page.length > 0 - ? prisma.documentThumbnail.findMany({ - where: { userId: user.id, documentId: { in: pageIds } }, - select: { documentId: true }, - }) - : Promise.resolve([]), - ]); - const indexSources = new Map(); - for (const row of indexedRows) indexSources.set(row.documentId, row.source); - const thumbnailIds = new Set(); - for (const row of thumbRows) thumbnailIds.add(row.documentId); - - annotate({ - action: { name: "documents.inbound.list" }, - meta: { - count: page.length, - sort, - order, - filtered: Boolean( - q || (kind && kind.length > 0) || episodeId || encounterId || year, - ), - }, - }); - - return apiSuccess({ - documents: page.map((doc) => - serialiseDocument( - doc, - factCounts.get(doc.id) ?? { factCount: 0, pendingCount: 0 }, - linkMap.get(doc.id) ?? [], - indexSources.has(doc.id), - toContentIndexSource(indexSources.get(doc.id)), - thumbnailIds.has(doc.id), - visitLinkMap.get(doc.id) ?? [], - ), - ), - nextCursor, - }); + return apiSuccess(await listVaultDocuments(user.id, parsed.data)); }); diff --git a/src/lib/documents/upload-answer.ts b/src/lib/documents/upload-answer.ts new file mode 100644 index 000000000..9e7276b08 --- /dev/null +++ b/src/lib/documents/upload-answer.ts @@ -0,0 +1,66 @@ +/** + * The stored row a vault upload answers with (`POST /api/documents/inbound`), + * serialised with its condition and visit links: a fresh insert with no facts + * yet, or the existing live row a duplicate resolved to, with its fact + * tallies. + */ +import { prisma } from "@/lib/db"; +import { + loadConditionLinks, + loadDocumentEncounterLinks, +} from "@/lib/documents/links"; +import { + serialiseDocument, + type SerialisableDocument, +} from "@/lib/documents/store"; + +/** The existing row a duplicate upload resolved to. */ +export async function duplicateDocumentBody( + userId: string, + existing: SerialisableDocument, +) { + const [links, visitLinks, groups] = await Promise.all([ + loadConditionLinks(userId, [existing.id]), + loadDocumentEncounterLinks(userId, [existing.id]), + prisma.extractedFact.groupBy({ + by: ["status"], + where: { userId, documentId: existing.id }, + _count: { _all: true }, + }), + ]); + let factCount = 0; + let pendingCount = 0; + for (const g of groups) { + if (g.status !== "REJECTED") factCount += g._count._all; + if (g.status === "PENDING") pendingCount += g._count._all; + } + return serialiseDocument( + existing, + { factCount, pendingCount }, + links.get(existing.id) ?? [], + false, + null, + false, + visitLinks.get(existing.id) ?? [], + ); +} + +/** A row the upload just stored. */ +export async function storedDocumentBody( + userId: string, + document: SerialisableDocument, +) { + const [links, visitLinks] = await Promise.all([ + loadConditionLinks(userId, [document.id]), + loadDocumentEncounterLinks(userId, [document.id]), + ]); + return serialiseDocument( + document, + { factCount: 0, pendingCount: 0 }, + links.get(document.id) ?? [], + false, + null, + false, + visitLinks.get(document.id) ?? [], + ); +} diff --git a/src/lib/documents/upload-request.ts b/src/lib/documents/upload-request.ts new file mode 100644 index 000000000..fdf00758d --- /dev/null +++ b/src/lib/documents/upload-request.ts @@ -0,0 +1,142 @@ +/** + * Reading one vault upload (`POST /api/documents/inbound`): the source key + * that may ride the query string, the bounded multipart body, the metadata + * fields beside the file, and the one source key the upload is filed under. + * Each step answers what it found; the route turns a refusal into its + * response. + */ +import type { z } from "zod/v4"; + +import { BodyTooLargeError, readBoundedBody } from "@/lib/labs/ocr-upload"; +import { + documentCreateSchema, + documentSourceKeySchema, + type DocumentSourceSystemValue, +} from "@/lib/validations/inbound-documents"; + +const UPLOAD_BODY_TIMEOUT_MS = 30_000; + +export interface QuerySourceKey { + sourceSystem: DocumentSourceSystemValue; + sourceId: string; + sourceInstance?: string; +} + +/** + * The source key in the query string. Both halves or neither; half a key is + * a mistake worth saying so about. + */ +export function parseQuerySourceKey( + url: URL, +): + | { kind: "none" } + | { kind: "invalid"; issues: z.core.$ZodIssue[] } + | { kind: "key"; key: QuerySourceKey } { + const querySystem = url.searchParams.get("sourceSystem"); + const queryId = url.searchParams.get("sourceId"); + const queryInstance = url.searchParams.get("sourceInstance"); + if (querySystem === null && queryId === null) return { kind: "none" }; + const parsedKey = documentSourceKeySchema.safeParse({ + sourceSystem: querySystem ?? undefined, + sourceId: queryId ?? undefined, + sourceInstance: queryInstance ?? undefined, + }); + if (!parsedKey.success) { + return { kind: "invalid", issues: parsedKey.error.issues }; + } + return { kind: "key", key: parsedKey.data }; +} + +/** + * Read the multipart body, aborting at `bodyCap` bytes or after the upload + * deadline, and parse it as form data. + */ +export async function readUploadForm( + request: Request, + bodyCap: number, +): Promise< + | { kind: "form"; formData: FormData } + | { kind: "tooLarge" } + | { kind: "timeout" } + | { kind: "invalid" } +> { + try { + const bytes = await readBoundedBody(request.body, bodyCap, { + signal: request.signal, + deadline: Date.now() + UPLOAD_BODY_TIMEOUT_MS, + }); + const formData = await new Request(request.url, { + method: "POST", + headers: { "content-type": request.headers.get("content-type") ?? "" }, + body: bytes, + }).formData(); + return { kind: "form", formData }; + } catch (err) { + if (err instanceof BodyTooLargeError) return { kind: "tooLarge" }; + if (err instanceof DOMException && err.name === "AbortError") { + return { kind: "timeout" }; + } + return { kind: "invalid" }; + } +} + +/** + * The optional metadata (title / kind / documentDate / episodeIds / + * encounterIds / source key / aiRead): the form fields beside the file. Both + * id lists may be repeated. + */ +export function parseUploadMetadata(formData: FormData) { + const rawEpisodeIds = formData + .getAll("episodeIds") + .filter((v): v is string => typeof v === "string"); + const rawEncounterIds = formData + .getAll("encounterIds") + .filter((v): v is string => typeof v === "string"); + return documentCreateSchema.safeParse({ + title: formData.get("title") ?? undefined, + kind: formData.get("kind") ?? undefined, + documentDate: formData.get("documentDate") ?? undefined, + episodeIds: rawEpisodeIds.length > 0 ? rawEpisodeIds : undefined, + encounterIds: rawEncounterIds.length > 0 ? rawEncounterIds : undefined, + sourceSystem: formData.get("sourceSystem") ?? undefined, + sourceId: formData.get("sourceId") ?? undefined, + sourceInstance: formData.get("sourceInstance") ?? undefined, + aiRead: formData.get("aiRead") ?? undefined, + }); +} + +/** + * The key from the query string, or from the form fields; if a caller sends + * both they have to agree. + */ +export function resolveUploadSourceKey( + queryKey: QuerySourceKey | null, + metadata: z.infer, +): + | { kind: "mismatch" } + | { + kind: "key"; + sourceSystem: DocumentSourceSystemValue | null; + sourceId: string | null; + sourceInstance: string | null; + } { + const formSystem = metadata.sourceSystem ?? null; + const formId = formSystem ? (metadata.sourceId ?? null) : null; + const formInstance = formId ? (metadata.sourceInstance ?? null) : null; + if ( + queryKey && + formSystem !== null && + (formSystem !== queryKey.sourceSystem || + formId !== queryKey.sourceId || + (formInstance !== null && + formInstance !== (queryKey.sourceInstance ?? null))) + ) { + return { kind: "mismatch" }; + } + const sourceSystem = queryKey?.sourceSystem ?? formSystem; + const sourceId = queryKey?.sourceId ?? formId; + const sourceInstance = sourceId + ? (queryKey?.sourceInstance ?? formInstance ?? null) + : null; + return { kind: "key", sourceSystem, sourceId, sourceInstance }; +} diff --git a/src/lib/documents/upload-responses.ts b/src/lib/documents/upload-responses.ts new file mode 100644 index 000000000..0dcaba140 --- /dev/null +++ b/src/lib/documents/upload-responses.ts @@ -0,0 +1,144 @@ +/** + * The answers a vault upload (`POST /api/documents/inbound`) gives besides a + * freshly stored row: the receipt a narrow token gets, a duplicate, a + * re-send of a deleted import, the alias limit, and the size refusal. + */ +import { NextResponse } from "next/server"; + +import { apiError } from "@/lib/api-response"; +import { annotate } from "@/lib/logging/context"; +import type { SerialisableDocument } from "@/lib/documents/store"; +import type { SourceKeyMatch } from "@/lib/documents/source-key"; +import { duplicateDocumentBody } from "@/lib/documents/upload-answer"; +import type { DocumentSourceSystemValue } from "@/lib/validations/inbound-documents"; + +/** §3.2 — 413 fileTooLarge with the configured limit in `meta`. */ +export function fileTooLarge(maxFileBytes: number): NextResponse { + return apiError("File is too large.", 413, { + errorCode: "documents.inbound.fileTooLarge", + reason: "fileTooLarge", + maxFileBytes, + }); +} + +/** What a single upload asked for, carried into the annotations. */ +export interface UploadContext { + userId: string; + /** A narrow `documents:write` token, not a session or a `["*"]` token. */ + scoped: boolean; + sourceSystem: DocumentSourceSystemValue | null; + aiDeferred: boolean; +} + +/** + * The receipt a scoped caller gets in place of the stored row: the id and + * whether the upload was a duplicate. A write-only credential holder who + * knows a file's bytes learns that it is stored, not its title, filename, + * links or fact counts. + */ +export function receiptResponse( + id: string | null, + status: 200 | 201, + flags: { duplicate: boolean; deleted?: boolean }, +): NextResponse { + const data = { + id, + duplicate: flags.duplicate, + ...(flags.deleted ? { deleted: true } : {}), + }; + return NextResponse.json( + { + data, + error: null, + ...(flags.duplicate + ? { + meta: { + duplicate: true, + ...(flags.deleted ? { deleted: true } : {}), + }, + } + : {}), + }, + { status }, + ); +} + +/** + * A re-send of an imported document the person deleted. + * 200 and no new row, for every caller: the deletion was a decision, and an + * importer re-running over the same source must not undo it. `id` is the + * tombstoned row while it exists and null once the purge took it; there is no + * row left to serialise either way, so the answer is the receipt. + */ +export function deletedResponse( + ctx: UploadContext, + id: string | null, +): NextResponse { + annotate({ + action: { name: "documents.vault.upload" }, + meta: { + documentId: id, + duplicate: true, + deleted: true, + scoped: ctx.scoped, + sourceSystem: ctx.sourceSystem, + aiDeferred: ctx.aiDeferred, + }, + }); + return receiptResponse(id, 200, { duplicate: true, deleted: true }); +} + +/** + * The same bytes have already been sent under too many source keys + * (`MAX_SOURCE_ALIASES_PER_DOCUMENT`). A conflict with what is stored, not a + * rate: nothing is remembered and nothing stored. + */ +export function aliasLimitResponse(): NextResponse { + return apiError( + "This file is already stored under too many source ids.", + 409, + { errorCode: "documents.inbound.sourceAliasLimit" }, + ); +} + +/** Answer a source key that is already held: duplicate, or deleted. */ +export function answerSourceKey( + ctx: UploadContext, + match: SourceKeyMatch, +): Promise | NextResponse { + return match.state === "live" + ? duplicateResponse(ctx, match.document) + : deletedResponse(ctx, match.id); +} + +/** + * §3.2 — a duplicate upload is NOT an error: return the existing live row + * with `meta.duplicate: true` at the envelope level (the UI toasts "already + * stored" and highlights the row). A scoped caller gets the receipt instead. + */ +export async function duplicateResponse( + ctx: UploadContext, + existing: SerialisableDocument, +): Promise { + annotate({ + action: { name: "documents.vault.upload" }, + meta: { + documentId: existing.id, + duplicate: true, + scoped: ctx.scoped, + sourceSystem: ctx.sourceSystem, + aiDeferred: ctx.aiDeferred, + }, + }); + if (ctx.scoped) { + return receiptResponse(existing.id, 200, { duplicate: true }); + } + return NextResponse.json( + { + data: await duplicateDocumentBody(ctx.userId, existing), + error: null, + meta: { duplicate: true }, + }, + { status: 200 }, + ); +} diff --git a/src/lib/documents/vault-list.ts b/src/lib/documents/vault-list.ts new file mode 100644 index 000000000..d1c813ba7 --- /dev/null +++ b/src/lib/documents/vault-list.ts @@ -0,0 +1,209 @@ +/** + * The vault list read behind `GET /api/documents/inbound`: title/filename + * search unioned with a whole-word content match, kind / episode / visit / + * year / date-range filters, sort, and keyset pagination. It NEVER selects + * the encrypted blob column. + */ +import type { z } from "zod/v4"; + +import { prisma } from "@/lib/db"; +import type { Prisma } from "@/generated/prisma/client"; +import { listPage } from "@/lib/http/list-query"; +import { annotate } from "@/lib/logging/context"; +import { hashQueryTokens } from "@/lib/documents/content-index"; +import { isoDateToUtc } from "@/lib/documents/ingest"; +import { + loadConditionLinks, + loadDocumentEncounterLinks, +} from "@/lib/documents/links"; +import { serialiseDocument } from "@/lib/documents/store"; +import { + toContentIndexSource, + type documentListQuerySchema, +} from "@/lib/validations/inbound-documents"; + +type VaultListQuery = z.infer; + +function listWhere( + userId: string, + query: VaultListQuery, +): Prisma.InboundDocumentWhereInput { + const { q, kind, episodeId, encounterId, year, from, to } = query; + const where: Prisma.InboundDocumentWhereInput = { + userId, + deletedAt: null, + }; + if (kind && kind.length > 0) where.kind = { in: kind }; + if (episodeId) where.conditionLinks = { some: { episodeId } }; + // The visit filter reads the SAME table the visit's own sheet writes, from + // the document side. Filtering rather than post-filtering: a page of fifty + // documents narrowed in Node would page wrongly. + if (encounterId) where.encounterLinks = { some: { encounterId } }; + if (year !== undefined) { + where.documentDate = { + gte: new Date(Date.UTC(year, 0, 1)), + lt: new Date(Date.UTC(year + 1, 0, 1)), + }; + } else if (from || to) { + where.documentDate = { + ...(from ? { gte: isoDateToUtc(from) } : {}), + // inclusive end-of-day for the `to` bound + ...(to ? { lt: new Date(isoDateToUtc(to).getTime() + 86_400_000) } : {}), + }; + } + if (q) { + // Substring match on the short plaintext fields … + const or: Prisma.InboundDocumentWhereInput[] = [ + { title: { contains: q, mode: "insensitive" } }, + { filename: { contains: q, mode: "insensitive" } }, + ]; + // … unioned with a WHOLE-WORD content match over the blind token index. + // The query is tokenised + HMAC'd the same way the index was built, then + // matched with a GIN-accelerated array-overlap (`hasSome` → `&&`). The + // list still never selects the encrypted text: only the opaque hashes + // are touched, in the related table. Degrades silently to title/filename + // when the caller has no indexed documents (no rows overlap). + const hashes = hashQueryTokens(q); + if (hashes.length > 0) { + or.push({ contentIndex: { is: { searchTokens: { hasSome: hashes } } } }); + } + where.OR = or; + } + return where; +} + +/** + * The sort column of a page; nullable sort columns sort nulls last so + * undated documents trail. + */ +function primaryOrder( + query: VaultListQuery, +): Prisma.InboundDocumentOrderByWithRelationInput { + const { sort, order } = query; + return sort === "documentDate" + ? { documentDate: { sort: order, nulls: "last" } } + : sort === "title" + ? { title: { sort: order, nulls: "last" } } + : { createdAt: order }; +} + +/** + * Fact tallies for the whole page in ONE grouped count: no per-document + * fan-out and no materialising fact-id rows. `factCount` counts every + * non-REJECTED fact (a rejected fact is discarded, not part of the + * document's tally), `pendingCount` the PENDING subset awaiting review. + */ +async function pageFactCounts( + userId: string, + pageIds: string[], +): Promise> { + const factCounts = new Map< + string, + { factCount: number; pendingCount: number } + >(); + if (pageIds.length === 0) return factCounts; + const groups = await prisma.extractedFact.groupBy({ + by: ["documentId", "status"], + where: { userId, documentId: { in: pageIds } }, + _count: { _all: true }, + }); + for (const g of groups) { + const entry = factCounts.get(g.documentId) ?? { + factCount: 0, + pendingCount: 0, + }; + const n = g._count._all; + if (g.status !== "REJECTED") entry.factCount += n; + if (g.status === "PENDING") entry.pendingCount += n; + factCounts.set(g.documentId, entry); + } + return factCounts; +} + +export async function listVaultDocuments( + userId: string, + query: VaultListQuery, +) { + const { q, kind, episodeId, encounterId, year, sort, order, cursor } = query; + const where = listWhere(userId, query); + // Keyset pagination on the sort column + a stable `id` tiebreak. + const orderBy: Prisma.InboundDocumentOrderByWithRelationInput[] = [ + primaryOrder(query), + { id: order }, + ]; + + // Hardening: the list NEVER fetches the encrypted blob column; a page of + // 50 rows would otherwise drag up to 50 × cap ciphertext bytes per request. + const { rows: page, cap } = await listPage(query.limit, (take) => + prisma.inboundDocument.findMany({ + where, + omit: { contentEncrypted: true }, + orderBy, + take, + ...(cursor ? { cursor: { id: cursor }, skip: 1 } : {}), + }), + ); + + const hasMore = cap.truncated; + const nextCursor = hasMore ? (page[page.length - 1]?.id ?? null) : null; + + const pageIds = page.map((d) => d.id); + const factCounts = await pageFactCounts(userId, pageIds); + + // Condition/visit links + the content-index and thumbnail probes for the + // page, all in ONE Promise.all: four independent grouped queries (no N+1, + // never a ciphertext/blob column), and the vault polls this list every few + // seconds so two serialized round-trips were pure added latency. + const [linkMap, visitLinkMap, indexedRows, thumbRows] = await Promise.all([ + loadConditionLinks(userId, pageIds), + loadDocumentEncounterLinks(userId, pageIds), + // Which of the page's documents have a content index (drives the + // searchable status + the provenance the UI reads to tell an AI-read + // document from a locally-indexed one). + page.length > 0 + ? prisma.documentContentIndex.findMany({ + where: { userId, documentId: { in: pageIds } }, + select: { documentId: true, source: true }, + }) + : Promise.resolve([]), + // Which of the page's documents have a preview thumbnail (gates the + // card's ). Grouped on the 1:1 side table. + page.length > 0 + ? prisma.documentThumbnail.findMany({ + where: { userId, documentId: { in: pageIds } }, + select: { documentId: true }, + }) + : Promise.resolve([]), + ]); + const indexSources = new Map(); + for (const row of indexedRows) indexSources.set(row.documentId, row.source); + const thumbnailIds = new Set(); + for (const row of thumbRows) thumbnailIds.add(row.documentId); + + annotate({ + action: { name: "documents.inbound.list" }, + meta: { + count: page.length, + sort, + order, + filtered: Boolean( + q || (kind && kind.length > 0) || episodeId || encounterId || year, + ), + }, + }); + + return { + documents: page.map((doc) => + serialiseDocument( + doc, + factCounts.get(doc.id) ?? { factCount: 0, pendingCount: 0 }, + linkMap.get(doc.id) ?? [], + indexSources.has(doc.id), + toContentIndexSource(indexSources.get(doc.id)), + thumbnailIds.has(doc.id), + visitLinkMap.get(doc.id) ?? [], + ), + ), + nextCursor, + }; +}