Merge branch 'feat/document-import-token' into release/v1.39.2

# Conflicts:
#	docs/api/openapi.yaml
#	docs/integrations/document-import.md

src/lib/documents/__tests__/index-document.test.ts

@@ -169,6 +169,21 @@ describe("indexDocumentContent — provider-first path", () => {
 });
 
 describe("indexDocumentContent — local fallback path", () => {
+  it("a deferred row stays local even when the job did not ask for it", async () => {
+    // #1038 — the row decides: an automatic index of a deferred document
+    // never reaches a provider, whichever path enqueued it.
+    vi.mocked(loadOwnedDocument).mockResolvedValue({
+      ...DOC,
+      aiReadDeferred: true,
+    } as never);
+    vi.mocked(resolveDocumentVisionProvider).mockResolvedValue(PICK as never);
+    visionOk();
+    const outcome = await indexDocumentContent("user-1", "doc-1");
+    expect(outcome).toMatchObject({ indexed: true, source: "local-pdf" });
+    expect(resolveDocumentVisionProvider).not.toHaveBeenCalled();
+    expect(transcribeDocument).not.toHaveBeenCalled();
+  });
+
   it("a deferred import stays local even with an external pick and the toggle ON", async () => {
     // #1038 — `aiRead=defer`: the person asked for AI reading to wait.
     vi.mocked(resolveDocumentVisionProvider).mockResolvedValue(PICK as never);

src/lib/documents/ai-route-support.ts

@@ -113,6 +113,11 @@ export interface LoadedDocument {
   contentCodec: string;
   mimeType: string;
   status: string;
+  /**
+   * The import asked for `aiRead=defer` (#1038). Automatic paths read the
+   * document locally only while it is set.
+   */
+  aiReadDeferred: boolean;
 }
 
 /** Load one live, owner-scoped document with the columns the AI actions need. */
@@ -129,6 +134,7 @@ export async function loadOwnedDocument(
       contentCodec: true,
       mimeType: true,
       status: true,
+      aiReadDeferred: true,
     },
   });
 }

src/lib/documents/index-document.ts

@@ -388,17 +388,20 @@ export async function indexDocumentContent(
   const document = await loadOwnedDocument(userId, documentId);
   if (!document) return { indexed: false, reason: "not-found" };
   // v1.39.2 (#1038) — an import that asked for `aiRead=defer` gets the
-  // provider-free text layer only. An empty context is the same shape "no
-  // provider configured" already produces, so the tree falls straight to the
-  // local path without resolving a chain it would not use.
-  const provider: ResolvedIndexProvider = options.localOnly
-    ? {
-        chain: [],
-        pick: null,
-        consentOk: false,
-        dailyCap: 0,
-        costOwner: "operator",
-      }
-    : await resolveIndexProvider(userId);
+  // provider-free text layer only. The row decides, not only the job that
+  // asked: any automatic index of a deferred document stays local until the
+  // person reads it with AI themselves. An empty context is the same shape
+  // "no provider configured" already produces, so the tree falls straight to
+  // the local path without resolving a chain it would not use.
+  const provider: ResolvedIndexProvider =
+    options.localOnly || document.aiReadDeferred
+      ? {
+          chain: [],
+          pick: null,
+          consentOk: false,
+          dailyCap: 0,
+          costOwner: "operator",
+        }
+      : await resolveIndexProvider(userId);
   return indexLoadedDocument(userId, document, provider);
 }

src/lib/documents/source-key.ts

@@ -15,6 +15,7 @@
  * route both ask here, so they cannot disagree about what a key means.
  */
 import { prisma } from "@/lib/db";
+import { checkRateLimit, type RateLimitResult } from "@/lib/rate-limit";
 import type { DocumentSourceSystemValue } from "@/lib/validations/inbound-documents";
 import type { SerialisableDocument } from "@/lib/documents/store";
 
@@ -54,6 +55,14 @@ export async function findSourceKey(
   return purged ? { state: "deleted", id: null } : null;
 }
 
+/**
+ * How many further keys one document may collect. Real archives hold a file
+ * under one or two ids; the cap stops a leaked token from growing the table
+ * by sending the same bytes under id after id. Past it the upload is refused
+ * (409) rather than answered without remembering the key.
+ */
+export const MAX_SOURCE_ALIASES_PER_DOCUMENT = 20;
+
 /**
  * Remember that `sourceId` in `sourceSystem` was answered with `documentId`
  * (same bytes, stored earlier). Without this the key would be forgotten, and
@@ -65,9 +74,36 @@ export async function rememberSourceAlias(
   documentId: string,
   sourceSystem: DocumentSourceSystemValue,
   sourceId: string,
-): Promise<void> {
-  await prisma.documentSourceAlias.createMany({
+): Promise<"remembered" | "known" | "limit"> {
+  const held = await prisma.documentSourceAlias.count({
+    where: { userId, documentId },
+  });
+  if (held >= MAX_SOURCE_ALIASES_PER_DOCUMENT) return "limit";
+  const { count } = await prisma.documentSourceAlias.createMany({
     data: [{ userId, documentId, sourceSystem, sourceId }],
     skipDuplicates: true,
   });
+  return count > 0 ? "remembered" : "known";
+}
+
+/**
+ * Lookups of a source key per caller per hour, shared by the lookup route and
+ * the upload's query-string check so neither is an unmetered way to ask.
+ * Generous: an importer asks once per document, and a re-sync of a few
+ * thousand documents has to fit in an hour. Keyed on the token for a
+ * `documents:write` caller, on the person otherwise.
+ */
+const SOURCE_LOOKUP_LIMIT_PER_HOUR = 5000;
+const SOURCE_LOOKUP_WINDOW_MS = 60 * 60 * 1000;
+
+export function checkSourceLookupRateLimit(
+  scoped: boolean,
+  tokenId: string,
+  userId: string,
+): Promise<RateLimitResult> {
+  return checkRateLimit(
+    scoped ? `documents-source:token:${tokenId}` : `documents-source:${userId}`,
+    SOURCE_LOOKUP_LIMIT_PER_HOUR,
+    SOURCE_LOOKUP_WINDOW_MS,
+  );
 }

src/lib/jobs/__tests__/document-content-index-backfill.test.ts

@@ -270,6 +270,10 @@ describe("runContentIndexBackfillForUser", () => {
       reason: "ok",
     });
     expect(upsertContentIndex).toHaveBeenCalledTimes(2);
+    // Imports held back from AI reading (#1038) are never candidates.
+    expect(
+      vi.mocked(prisma.inboundDocument.findMany).mock.calls[0]?.[0]?.where,
+    ).toMatchObject({ aiReadDeferred: false });
   });
 
   it("includes PDFs as candidates for an image-input provider and rasterises them", async () => {

src/lib/jobs/document-content-index-backfill.ts

@@ -231,6 +231,9 @@ export async function runContentIndexBackfillForUser(
         deletedAt: null,
         mimeType: { in: candidateMimes },
         contentIndex: { is: null },
+        // An import held back from AI reading (#1038) is read only when the
+        // person opens it and asks; "index all" leaves it local.
+        aiReadDeferred: false,
       },
       select: { id: true },
       orderBy: { id: "asc" },

src/lib/openapi/error-codes.ts

@@ -132,6 +132,7 @@ export const ERROR_CODE_CATALOGUE: Readonly<Record<string, readonly string[]>> =
       "documents.inbound.rateLimited",
       "documents.inbound.restoreDuplicate",
       "documents.inbound.restoreGone",
+      "documents.inbound.sourceAliasLimit",
       "documents.inbound.uploadBusy",
       "documents.inbound.uploadTimeout",
       "documents.inbound.vaccinationNotFound",

src/lib/openapi/routes/documents.ts

@@ -407,7 +407,7 @@ export const inboundDocumentPaths: NonNullable<ZodOpenApiObject["paths"]> = {
           required: false,
           schema: { type: "string", enum: [...DOCUMENT_SOURCE_SYSTEMS] },
           description:
-            "Import source key, first half. Sent here instead of (or agreeing with) the form field, it is checked before the body is read or the hourly allowance is charged.",
+            "Import source key, first half. Sent here (and, if also in the form, agreeing with it), a key the vault already holds is answered at once: the file is not stored or processed and the request does not count toward the hourly upload limit, only toward the lookup allowance shared with `GET /api/documents/inbound/source`.",
         },
         {
           name: "sourceId",
@@ -422,7 +422,7 @@ export const inboundDocumentPaths: NonNullable<ZodOpenApiObject["paths"]> = {
       summary: "Store a document (no extraction)",
       description:
         'STORE-ONLY upload. Stores the raw document ENCRYPTED at rest with `status: STORED` and runs NO extraction — provider-free, no AI consent / budget / egress. A file is always filable, even with no document-scan provider configured. `multipart/form-data`: a `file` plus optional `title`, `kind`, `documentDate` (YYYY-MM-DD), and repeated `episodeIds` form fields (pre-link to the caller\'s illness/condition episodes). Accepted types (magic-byte sniffed, never the wire Content-Type): PDF/JPEG/PNG/WebP/GIF render inline; Office (docx/xlsx/pptx/doc/xls/ppt), text/CSV/Markdown/RTF, TIFF, HEIC/HEIF, XML/JSON are stored verbatim and served download-only. HEIC is stored as-is but attachment-only — prefer transcoding to JPEG client-side for inline preview parity. Error contract: `413` with `meta.reason = "fileTooLarge"` (+ `maxFileBytes`) or `"quotaExceeded"` (+ `quotaBytes`, `usedBytes`); `415` with `meta.reason = "unsupportedType"`. A same-user duplicate (same bytes) returns 200 + `meta.duplicate: true` with the existing row — not an error. `Idempotency-Key` honoured. Read `GET /api/documents/inbound/usage` for the effective limits before offering an upload. AI extraction is a separate opt-in action — see `POST /api/documents/inbound/{id}/extract`.\n\n' +
-        "**Importing from another system (v1.39.2).** Optional `sourceSystem` (`PAPERLESS`, `PAPRA`, `OTHER`) and `sourceId` (its id there, printable, up to 128 characters; needs `sourceSystem`) key the upload. A re-send with a known key is a duplicate: 200 with the live row (`meta.duplicate: true`), or, when the owner DELETED that document, 200 with `data.deleted: true` and `meta.deleted: true` and nothing stored; this holds after the 30-day purge too. The key may instead ride the query string (`?sourceSystem=…&sourceId=…`): it is then answered before the body is read and before the hourly allowance is charged, so a re-send of a document already held costs neither; a key in both places must agree (else 422). A key that is answered with an existing document by its bytes is remembered for that document, so deleting it keeps that key deleted too. `OTHER` is one shared namespace per account: two different systems both sent as `OTHER` must not reuse each other\'s ids. An upload answered as a duplicate or as deleted hands its slot of the hourly allowance back. `aiRead=defer` holds back automatic AI reading for this upload: the thumbnail and a local text index still run, the summary and lab staging do not, and the document stays out of the summary catch-up until the person reads it with AI or generates its summary. Absent, behaviour is unchanged.\n\n" +
+        "**Importing from another system (v1.39.2).** Optional `sourceSystem` (`PAPERLESS`, `PAPRA`, `OTHER`) and `sourceId` (its id there, printable, up to 128 characters; needs `sourceSystem`) key the upload. A re-send with a known key is a duplicate: 200 with the live row (`meta.duplicate: true`), or, when the owner DELETED that document, 200 with `data.deleted: true` and `meta.deleted: true` and nothing stored; this holds after the 30-day purge too. The key may also ride the query string (`?sourceSystem=…&sourceId=…`): a key the vault already holds is then answered at once, the file is not stored or processed, and the request counts only toward the lookup allowance shared with `GET /api/documents/inbound/source` (5000 an hour per token, per person for a session; 429 past it), not toward the hourly upload limit. A key in both places must agree (else 422). Every other upload counts toward the hourly upload limit, duplicates included. A key that is answered with an existing document by its bytes is remembered for that document, so deleting it keeps that key deleted too; one document takes at most 20 such keys, beyond which the upload is refused 409 (`documents.inbound.sourceAliasLimit`). `OTHER` is one shared namespace per account: two different systems both sent as `OTHER` must not reuse each other\'s ids. `aiRead=defer` holds back automatic AI reading for this upload: the thumbnail and a local text index still run, the summary and lab staging do not, and the document stays out of every automatic AI path (the summary catch-up, the automatic index, Index all for search) until the person reads it with AI or generates its summary. Absent, behaviour is unchanged.\n\n" +
         "**Narrow token.** Also reachable with a `documents:write` Bearer (minted at `POST /api/tokens/documents`), which reaches only this route and the source-key lookup `GET /api/documents/inbound/source`. Such a caller draws on a bucket of its own, keyed on the token (default 120 an hour, `DOCUMENT_UPLOAD_LIMIT_PER_HOUR`, clamped 1-1000; the 429 carries `Retry-After` and `X-RateLimit-*`), and gets a `DocumentUploadReceipt` instead of the stored row; its 413 `quotaExceeded` carries no `quotaBytes` / `usedBytes`. The response is an untagged union (row or receipt) on purpose: a discriminator would need a new field on the row every existing client decodes. A cookie or wildcard caller keeps the 60-an-hour per-user bucket and the full row. With the vault module off the answer is 403 `module.disabled` for every caller.",
       requestBody: {
         required: true,
@@ -494,6 +494,11 @@ export const inboundDocumentPaths: NonNullable<ZodOpenApiObject["paths"]> = {
             "Too large: `meta.reason` is `fileTooLarge` (with `maxFileBytes`) or `quotaExceeded` (with `quotaBytes` + `usedBytes`).",
           content: { "application/json": { schema: errorEnvelope } },
         },
+        "409": {
+          description:
+            "The same bytes are already stored under 20 other import source ids (`documents.inbound.sourceAliasLimit`). Nothing was stored or remembered.",
+          content: { "application/json": { schema: errorEnvelope } },
+        },
         "415": {
           description:
             'Unsupported or unidentifiable type (`meta.reason = "unsupportedType"`). Executables, HTML/SVG, and generic archives are always refused.',
@@ -508,7 +513,7 @@ export const inboundDocumentPaths: NonNullable<ZodOpenApiObject["paths"]> = {
       tags: ["Documents"],
       summary: "Look up an import source key",
       description:
-        "Whether the vault already holds a document under this import source key, so an importer can skip a document before downloading it from the source system. Answers exactly what an upload with the same key would: `known`, the document `id` (null once a deleted document was purged), and whether the owner `deleted` it. Checks the document's own key, keys remembered for it when an import sent the same bytes under another key, and the keys of purged documents. Reachable with a `documents:write` Bearer (the scope's second and last route) or a cookie session; stores nothing. 5000 lookups an hour per token (per person for a session); gated on the `inboundDocuments` module.",
+        "Whether the vault already holds a document under this import source key, so an importer can skip a document before downloading it from the source system. Answers exactly what an upload with the same key would: `known`, the document `id` (null once a deleted document was purged), and whether the owner `deleted` it. Checks the document's own key, keys remembered for it when an import sent the same bytes under another key, and the keys of purged documents. Reachable with a `documents:write` Bearer (the scope's second and last route) or a cookie session; stores nothing. 5000 lookups an hour per token (per person for a session), shared with the upload's query-string key check; gated on the `inboundDocuments` module.",
       parameters: [
         {
           name: "sourceSystem",