diff --git a/.env.production.example b/.env.production.example index fe3144723..edb068b76 100644 --- a/.env.production.example +++ b/.env.production.example @@ -290,6 +290,14 @@ API_TOKEN_HMAC_KEY="" # docs/integrations/ai-providers.md. # DOCUMENT_AI_LIMIT_PER_HOUR="" +# Hourly upload ceiling for each document-upload token (optional). Applies +# only to uploads made with a token minted under Settings > API > Document +# tokens: a Paperless workflow or the import script. Default 120, clamped to +# 1-1000. Uploads from the web app and the phone keep their own bucket. Raise +# it to import a large archive faster; the per-user storage quota still +# applies. See docs/integrations/document-import.md. +# DOCUMENT_UPLOAD_LIMIT_PER_HOUR="" + # Per-user hourly ceiling for lab-report scans (optional). The photo/PDF scan # and the browser-OCR text scan share one bucket per user. Default 6, clamped # to 1-1000. Raise it when users scan a stack of reports at once against a diff --git a/docker-compose.yml b/docker-compose.yml index 4e0897edb..0a0adc15d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -290,6 +290,12 @@ x-healthlog-env: &healthlog_env # (the `environment:` block is a whitelist). See # docs/integrations/ai-providers.md. DOCUMENT_AI_LIMIT_PER_HOUR: "${DOCUMENT_AI_LIMIT_PER_HOUR:-}" + # Hourly upload ceiling for each narrow document-upload token (a Paperless + # workflow, the import script; optional; default 120, clamped 1-1000). + # Uploads from the web app and the phone keep their own bucket. Listed here + # so an operator value in .env reaches the container. See + # docs/integrations/document-import.md. + DOCUMENT_UPLOAD_LIMIT_PER_HOUR: "${DOCUMENT_UPLOAD_LIMIT_PER_HOUR:-}" # Per-user hourly ceiling for lab-report scans (photo/PDF and browser-OCR # text share one bucket; optional; default 6, clamped 1-1000). A scan that # fails before the provider is called hands its slot back. Listed here so an diff --git a/docs/api/openapi.yaml b/docs/api/openapi.yaml index 8e942b3e6..7f26349f3 100644 --- a/docs/api/openapi.yaml +++ b/docs/api/openapi.yaml @@ -2807,10 +2807,11 @@ paths: is no GENERIC mint any more — the POST at this exact path issued `["medication:ingest"]` and was removed, because that scope reached no ingest route while the pre-fail-closed default let it reach everything else. The credentials that work are minted where they are scoped: the per-medication API-endpoint toggle, `POST - /api/mcp/tokens` for a connector, and the sibling `POST /api/tokens/measurements` for third-party measurement - ingest. And unlike the revoke, this read is NOT gated on the operator's instance-wide API switch: that switch - governs the surfaces a token is for, not a token's ability to authenticate, so tokens stay live while it is off - and their owner has to be able to see them. + /api/mcp/tokens` for a connector, the sibling `POST /api/tokens/measurements` for third-party measurement + ingest, and `POST /api/tokens/documents` for document upload from another system. And unlike the revoke, this + read is NOT gated on the operator's instance-wide API switch: that switch governs the surfaces a token is for, + not a token's ability to authenticate, so tokens stay live while it is off and their owner has to be able to see + them. responses: "200": description: The caller's tokens, newest first. @@ -2916,6 +2917,91 @@ paths: application/json: schema: $ref: "#/components/schemas/ErrorEnvelope" + /api/tokens/documents: + post: + tags: + - Auth + summary: Mint a document-upload token + description: >- + Mints a Bearer scoped to exactly `documents:write` and audits the mint. THE RESPONSE CARRIES THE RAW TOKEN, + once; it is stored as an HMAC and no path re-reveals it. + + + **What it can do.** `POST /api/documents/inbound` on its owner's own record, and nothing else. It is the door + another document system pushes through: a Paperless-ngx workflow posting a newly tagged document, or an import + script copying an archive. Uploads through it draw on a bucket of their own, keyed on the token (default 120 an + hour, operator-tunable via `DOCUMENT_UPLOAD_LIMIT_PER_HOUR`), and get back a receipt (`id`, `duplicate`, and + `deleted` when the source key belongs to a document the owner deleted) rather than the stored row. + + + **What it cannot do.** List, read, download, preview, edit, delete or bulk-act on a document, reach the + document-AI routes, act on a shared record, or mint another token. Every other route names no scope and refuses + it 403. + + + Minting requires a COOKIE SESSION, for the reason the measurement mint gives: a day-lived native access token + must not be able to leave behind a year-lived credential. Gated by the operator's instance-wide API switch. Body + capped at 16 KiB; 10 mints per user per minute, and at most 10 live document tokens held at once. Tokens appear + in `GET /api/tokens` and are revoked at `DELETE /api/tokens/{id}`. + requestBody: + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/CreateDocumentTokenRequest" + responses: + "201": + description: Token minted. The `token` field is the only copy of the secret. + content: + application/json: + schema: + $ref: "#/components/schemas/CreateDocumentTokenEnvelope" + "400": + description: Body is not parseable JSON. + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorEnvelope" + "401": *a1 + "403": + description: The operator has switched the API off instance-wide. Nothing was minted. A Bearer caller is refused 401 + before the switch is read. + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorEnvelope" + "409": + description: The account already holds 10 live document tokens (`meta.errorCode` = `tokens.documents.ceiling_reached`). + Nothing was minted. Revoke one at `DELETE /api/tokens/{id}`; only live tokens carrying `documents:write` + count. + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorEnvelope" + "413": + description: Body exceeds 16 KiB. + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorEnvelope" + "415": + description: Content-Type is not `application/json`. + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorEnvelope" + "422": + description: Validation failed. The envelope carries every offending issue. + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorEnvelope" + "429": + description: More than 10 mints from this account in a minute. Nothing was minted. + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorEnvelope" /api/tokens/{id}: delete: tags: @@ -15572,18 +15658,36 @@ paths: tags: - Documents summary: Store a document (no extraction) - description: "STORE-ONLY upload. Stores the raw document ENCRYPTED at rest with `status: STORED` and runs NO extraction - — provider-free, no AI consent / budget / egress. A file is always filable, even with no document-scan provider + description: >- + STORE-ONLY upload. Stores the raw document ENCRYPTED at rest with `status: STORED` and runs NO extraction — + provider-free, no AI consent / budget / egress. A file is always filable, even with no document-scan provider configured. `multipart/form-data`: a `file` plus optional `title`, `kind`, `documentDate` (YYYY-MM-DD), and repeated `episodeIds` form fields (pre-link to the caller's illness/condition episodes). Accepted types (magic-byte sniffed, never the wire Content-Type): PDF/JPEG/PNG/WebP/GIF render inline; Office (docx/xlsx/pptx/doc/xls/ppt), text/CSV/Markdown/RTF, TIFF, HEIC/HEIF, XML/JSON are stored verbatim and served download-only. HEIC is stored as-is but attachment-only — prefer transcoding to JPEG client-side for inline - preview parity. Error contract: `413` with `meta.reason = \"fileTooLarge\"` (+ `maxFileBytes`) or - `\"quotaExceeded\"` (+ `quotaBytes`, `usedBytes`); `415` with `meta.reason = \"unsupportedType\"`. A same-user + preview parity. Error contract: `413` with `meta.reason = "fileTooLarge"` (+ `maxFileBytes`) or + `"quotaExceeded"` (+ `quotaBytes`, `usedBytes`); `415` with `meta.reason = "unsupportedType"`. A same-user duplicate (same bytes) returns 200 + `meta.duplicate: true` with the existing row — not an error. `Idempotency-Key` honoured. Read `GET /api/documents/inbound/usage` for the effective limits before offering an - upload. AI extraction is a separate opt-in action — see `POST /api/documents/inbound/{id}/extract`." + upload. AI extraction is a separate opt-in action — see `POST /api/documents/inbound/{id}/extract`. + + + **Importing from another system (v1.39.2).** Optional `sourceSystem` (`PAPERLESS`, `PAPRA`, `OTHER`) and + `sourceId` (its id there, printable, up to 128 characters; needs `sourceSystem`) key the upload. A re-send with + a known key is a duplicate: 200 with the live row (`meta.duplicate: true`), or, when the owner DELETED that + document, 200 with `data.deleted: true` and `meta.deleted: true` and nothing stored; this holds after the 30-day + purge too. `aiRead=defer` holds back automatic AI reading for this upload: the thumbnail and a local text index + still run, the summary and lab staging do not, so the document waits for the person to read it deliberately. + Absent, behaviour is unchanged. + + + **Narrow token.** Also reachable with a `documents:write` Bearer (minted at `POST /api/tokens/documents`), and + it is the only route that scope reaches. Such a caller draws on a bucket of its own, keyed on the token (default + 120 an hour, `DOCUMENT_UPLOAD_LIMIT_PER_HOUR`, clamped 1-1000; the 429 carries `Retry-After` and + `X-RateLimit-*`), and gets a `DocumentUploadReceipt` instead of the stored row. A cookie or wildcard caller + keeps the 60-an-hour per-user bucket and the full row. With the vault module off the answer is 403 + `module.disabled` for every caller. parameters: - *a6 requestBody: @@ -15628,22 +15732,41 @@ paths: type: array items: type: string + sourceSystem: + description: Optional. The system an imported document comes from. Shown on the detail view. + type: string + enum: + - PAPERLESS + - PAPRA + - OTHER + sourceId: + description: Optional. The document's id in `sourceSystem`; requires it. Unique per account across live and deleted + documents. + type: string + maxLength: 128 + aiRead: + description: Optional. `defer` holds back automatic AI reading for this upload (local index and thumbnail only). + type: string + enum: + - defer required: - file responses: "200": - description: "Duplicate — the same bytes are already stored: the existing row, with `meta.duplicate: true` at the - envelope level." + description: "Duplicate — the same bytes, or the same source key, are already stored: the existing row (a + `DocumentUploadReceipt` for a `documents:write` token), with `meta.duplicate: true` at the envelope level. + When the source key belongs to a document the owner deleted: a receipt with `deleted: true` and + `meta.deleted: true`, nothing stored." content: application/json: schema: - $ref: "#/components/schemas/InboundDocumentEnvelope" + $ref: "#/components/schemas/DocumentUploadEnvelope" "201": - description: The stored document. + description: The stored document (a `DocumentUploadReceipt` for a `documents:write` token). content: application/json: schema: - $ref: "#/components/schemas/InboundDocumentEnvelope" + $ref: "#/components/schemas/DocumentUploadEnvelope" "401": *a1 "409": *a7 "413": @@ -21981,6 +22104,21 @@ components: permission array is a literal, so no request can widen it. `name` is what the token is listed under — name it after the thing you are pasting it into, because that is what you will be reading when you decide whether to revoke it. `expiresInDays` defaults to 365." + CreateDocumentTokenRequest: + type: object + properties: + name: + type: string + minLength: 1 + maxLength: 100 + expiresInDays: + type: integer + minimum: 1 + maximum: 365 + required: + - name + description: "Mint a document-upload Bearer. No `scope` field: the endpoint mints one shape. `name` is what the token is + listed under; name it after the system it is pasted into. `expiresInDays` defaults to 365." RegisterRequest: type: object properties: @@ -30586,16 +30724,17 @@ components: sharing.renounce.not_found, sharing.revoke.already_ended, sharing.revoke.not_found, sharing.session.changed, sharing.switch.invalid, sharing.switch.wrong_transport; source-priority — source-priority.body.invalid_json, source-priority.body.invalid_shape; tokens — - tokens.measurements.ceiling_reached; vaccination — vaccination.booster-invalid, - vaccination.booster-no-antigen, vaccination.booster-out-of-scope, vaccination.encounter-not-found, - vaccination.identity-required, vaccination.invalid, vaccination.link.invalid, vaccination.not-found, - vaccination.practitioner-not-found; workout — workout.batch.invalid, workout.batch.payload_too_large, - workout.batch.too_large; (unprefixed) — about_me_conflict, ai_response_truncated, coach_prefs_conflict, - dashboard_layout_conflict, empty_file, health_score_config_conflict, insights_layout_conflict, - invalid_base_updated_at, json_carries_no_intake_time, json_not_an_array, medication_layout_conflict, - missing_required_columns, modules_conflict, mood_tag_layout_conflict, not_configured, - notification_prefs_conflict, oidc_only, private_origin_not_approved, private_origin_not_grantable, - rate_limited_self, too_many_rows, unreadable_json, upstream_invalid_json, vapid_not_configured" + tokens.documents.ceiling_reached, tokens.measurements.ceiling_reached; vaccination — + vaccination.booster-invalid, vaccination.booster-no-antigen, vaccination.booster-out-of-scope, + vaccination.encounter-not-found, vaccination.identity-required, vaccination.invalid, + vaccination.link.invalid, vaccination.not-found, vaccination.practitioner-not-found; workout — + workout.batch.invalid, workout.batch.payload_too_large, workout.batch.too_large; (unprefixed) — + about_me_conflict, ai_response_truncated, coach_prefs_conflict, dashboard_layout_conflict, empty_file, + health_score_config_conflict, insights_layout_conflict, invalid_base_updated_at, + json_carries_no_intake_time, json_not_an_array, medication_layout_conflict, missing_required_columns, + modules_conflict, mood_tag_layout_conflict, not_configured, notification_prefs_conflict, oidc_only, + private_origin_not_approved, private_origin_not_grantable, rate_limited_self, too_many_rows, + unreadable_json, upstream_invalid_json, vapid_not_configured" type: string capability: description: "On an AI refusal: the capability that is unavailable, one of the keys of `AiCapabilities` (`coach`, @@ -35068,6 +35207,19 @@ components: anyOf: - type: string - type: "null" + sourceSystem: + anyOf: + - type: string + enum: + - PAPERLESS + - PAPRA + - OTHER + - type: "null" + sourceId: + anyOf: + - type: string + maxLength: 128 + - type: "null" summary: anyOf: - type: string @@ -38407,6 +38559,42 @@ components: - expiresAt additionalProperties: false description: The one response that carries a usable measurement-ingest Bearer value. + CreateDocumentTokenEnvelope: + type: object + properties: + data: + $ref: "#/components/schemas/CreateDocumentTokenResponse" + error: + type: "null" + meta: + type: object + properties: + requestId: + type: string + additionalProperties: false + required: + - data + - error + additionalProperties: false + CreateDocumentTokenResponse: + type: object + properties: + token: + type: string + description: The raw `hlk_` Bearer, RETURNED EXACTLY ONCE. It is stored only as an HMAC-SHA256 hash and cannot be + retrieved later. Treat it as a secret. + name: + type: string + expiresAt: + type: string + format: date-time + pattern: ^(?:(?:\d\d[2468][048]|\d\d[13579][26]|\d\d0[48]|[02468][048]00|[13579][26]00)-02-29|\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\d|30)|(?:02)-(?:0[1-9]|1\d|2[0-8])))T(?:(?:[01]\d|2[0-3]):[0-5]\d(?::[0-5]\d(?:\.\d+)?)?(?:Z|([+-](?:[01]\d|2[0-3]):[0-5]\d)))$ + required: + - token + - name + - expiresAt + additionalProperties: false + description: The one response that carries a usable document-upload Bearer value. ApiTokenRevokeEnvelope: type: object properties: @@ -63326,6 +63514,18 @@ components: - type: "null" hasThumbnail: type: boolean + sourceSystem: + anyOf: + - type: string + enum: + - PAPERLESS + - PAPRA + - OTHER + - type: "null" + sourceId: + anyOf: + - type: string + - type: "null" createdAt: type: string updatedAt: @@ -63352,6 +63552,8 @@ components: - lastIndexAttemptAt - lastIndexOutcome - hasThumbnail + - sourceSystem + - sourceId - createdAt - updatedAt additionalProperties: false @@ -63366,7 +63568,9 @@ components: attempt finished (successful or not; null when none ran); `lastIndexOutcome` says why that attempt produced no index, and is null when the attempt succeeded or none ran. `hasThumbnail` is true when a preview thumbnail has been rendered in the background (fetch it from `/api/documents/inbound/{id}/thumbnail`); false means no preview - yet or an unsupported type — show a placeholder. Treat an unknown `kind` value as OTHER when decoding. + yet or an unsupported type — show a placeholder. `sourceSystem` / `sourceId` say where an imported document came + from and its id there (both null for a document added in HealthLog); treat an unknown `sourceSystem` as OTHER + when decoding. Treat an unknown `kind` value as OTHER when decoding. DocumentConditionLink: type: object properties: @@ -63399,11 +63603,13 @@ components: visit-kind ENUM CONSTANT rather than a rendered name: unlike a condition label, which is the person's own words, a visit kind is one of eight closed values whose human name the reader's own bundle owns, so publishing the constant is what lets a German reader and an English one each see it named correctly." - InboundDocumentEnvelope: + DocumentUploadEnvelope: type: object properties: data: - $ref: "#/components/schemas/InboundDocument" + anyOf: + - $ref: "#/components/schemas/InboundDocument" + - $ref: "#/components/schemas/DocumentUploadReceipt" error: type: "null" meta: @@ -63416,6 +63622,27 @@ components: - data - error additionalProperties: false + DocumentUploadReceipt: + type: object + properties: + id: + anyOf: + - type: string + - type: "null" + duplicate: + type: boolean + deleted: + type: boolean + const: true + required: + - id + - duplicate + additionalProperties: false + description: "What an upload answers instead of the stored row. A narrow `documents:write` token always gets this shape + (`duplicate: false` on 201, `true` on 200): a credential that can only add files learns whether a file is + stored, not its title, filename or links. Every caller gets it when the upload's source key belongs to a + document the owner DELETED (`deleted: true`, 200, nothing stored); `id` is the tombstoned row, or null once the + 30-day purge removed it." DocumentAiCapabilityEnvelope: type: object properties: @@ -63700,6 +63927,18 @@ components: - type: "null" hasThumbnail: type: boolean + sourceSystem: + anyOf: + - type: string + enum: + - PAPERLESS + - PAPRA + - OTHER + - type: "null" + sourceId: + anyOf: + - type: string + - type: "null" createdAt: type: string updatedAt: @@ -63755,6 +63994,8 @@ components: - lastIndexAttemptAt - lastIndexOutcome - hasThumbnail + - sourceSystem + - sourceId - createdAt - updatedAt - facts diff --git a/messages/de.json b/messages/de.json index f80638722..9c1082c04 100644 --- a/messages/de.json +++ b/messages/de.json @@ -6140,7 +6140,14 @@ "scopeNote": "Es kann ausschließlich neue Werte in deine eigene Akte eintragen — deine Daten nicht lesen, nichts ändern oder löschen und kein weiteres Token erstellen. Werte werden als von einem externen Gerät gekennzeichnet, damit du sie später erkennst, und lassen sich jederzeit korrigieren.", "createFailed": "Das Token konnte nicht erstellt werden. Bitte versuche es erneut." }, - "tokenMintMovedDescription": "Tokens werden dort ausgestellt, wo sie gelten: oben für Messwerte, oder im API-Endpoint eines Medikaments für genau dieses Medikament.", + "documentsToken": { + "title": "Dokument-Tokens", + "description": "Erzeuge ein Token, damit Paperless-ngx, Papra oder das Import-Skript Dokumente in deine Ablage legen können.", + "detail": "Das Token wird nur einmal angezeigt.", + "scopeNote": "Es kann ausschließlich neue Dokumente in deine eigene Ablage legen. Es kann nichts auflisten, öffnen, herunterladen, ändern oder löschen und kein weiteres Token erstellen. Zurück bekommt es nur, ob eine Datei gespeichert wurde, nicht wie du sie abgelegt hast.", + "createFailed": "Das Token konnte nicht erstellt werden. Bitte versuche es erneut." + }, + "tokenMintMovedDescription": "Tokens werden dort ausgestellt, wo sie gelten: oben für Messwerte und Dokumente, oder im API-Endpoint eines Medikaments für genau dieses Medikament.", "tokenNamePlaceholder": "Token-Name (z.B. iPhone Shortcut)", "tokenCreated": "Token erstellt — kopiere ihn jetzt! Er wird nicht erneut angezeigt.", "tokenCopy": "Token kopieren", @@ -9984,6 +9991,9 @@ "noConditions": "Mit keiner Erkrankung verknüpft", "previewUnavailable": "Keine Inline-Vorschau für diesen Dateityp", "uploadedMeta": "Hinzugefügt am {date} · {size}", + "importedFrom": "Importiert aus {system}", + "importedFromWithId": "Importiert aus {system} · ID {id}", + "sourceOther": "einem anderen System", "share": "Teilen", "labValuesLink": "{count} Laborwerte ansehen", "visitsLabel": "Gehört zu Termin", diff --git a/messages/en.json b/messages/en.json index ec7b4cbdf..a1be3a8bb 100644 --- a/messages/en.json +++ b/messages/en.json @@ -6140,7 +6140,14 @@ "scopeNote": "It can only add new readings to your own record — it cannot read your data, change or delete anything, or create another token. Readings are marked as coming from an external device, so you can tell them apart later, and you can still correct them yourself.", "createFailed": "The token could not be created. Please try again." }, - "tokenMintMovedDescription": "Tokens are issued where they apply: above for measurements, or in a medication's own API endpoint for that one medication.", + "documentsToken": { + "title": "Document tokens", + "description": "Mint a token so Paperless-ngx, Papra or the import script can add documents to your vault.", + "detail": "The token is shown once.", + "scopeNote": "It can only add new documents to your own vault. It cannot list, open, download, change or delete anything, and it cannot create another token. What it gets back is whether a file was stored, not how you filed it.", + "createFailed": "The token could not be created. Please try again." + }, + "tokenMintMovedDescription": "Tokens are issued where they apply: above for measurements and documents, or in a medication's own API endpoint for that one medication.", "tokenNamePlaceholder": "Token name (e.g., iPhone Shortcut)", "tokenCreated": "Token created — copy it now! It won't be shown again.", "tokenCopy": "Copy token", @@ -9984,6 +9991,9 @@ "noConditions": "Not linked to a condition", "previewUnavailable": "No inline preview for this file type", "uploadedMeta": "Added {date} · {size}", + "importedFrom": "Imported from {system}", + "importedFromWithId": "Imported from {system} · ID {id}", + "sourceOther": "another system", "share": "Share", "labValuesLink": "View {count} lab values", "visitsLabel": "Belongs to visit", diff --git a/messages/es.json b/messages/es.json index b64f58204..b89757989 100644 --- a/messages/es.json +++ b/messages/es.json @@ -6140,7 +6140,14 @@ "scopeNote": "Solo puede añadir mediciones nuevas a tu propio historial: no puede leer tus datos, ni modificar o borrar nada, ni crear otro token. Las mediciones se marcan como procedentes de un dispositivo externo, así las distingues más tarde, y puedes corregirlas igualmente.", "createFailed": "No se pudo crear el token. Inténtalo de nuevo." }, - "tokenMintMovedDescription": "Los tokens se emiten donde se aplican: arriba para las mediciones, o en el endpoint de API de un medicamento para ese medicamento concreto.", + "documentsToken": { + "title": "Tokens de documentos", + "description": "Crea un token para que Paperless-ngx, Papra o el script de importación añadan documentos a tu archivo.", + "detail": "El token se muestra una sola vez.", + "scopeNote": "Solo puede añadir documentos nuevos a tu propio archivo. No puede listar, abrir, descargar, modificar ni borrar nada, ni crear otro token. Lo único que recibe es si un archivo se guardó, no cómo lo archivaste.", + "createFailed": "No se pudo crear el token. Inténtalo de nuevo." + }, + "tokenMintMovedDescription": "Los tokens se emiten donde se aplican: arriba para las mediciones y los documentos, o en el endpoint de API de un medicamento para ese medicamento concreto.", "tokenNamePlaceholder": "Nombre del token (p. ej., Atajo de iPhone)", "tokenCreated": "Token creado — ¡cópialo ahora! No se mostrará de nuevo.", "tokenCopy": "Copiar token", @@ -9984,6 +9991,9 @@ "noConditions": "Sin vincular a una afección", "previewUnavailable": "Sin vista previa integrada para este tipo de archivo", "uploadedMeta": "Añadido el {date} · {size}", + "importedFrom": "Importado de {system}", + "importedFromWithId": "Importado de {system} · ID {id}", + "sourceOther": "otro sistema", "share": "Compartir", "labValuesLink": "Ver {count} valores de laboratorio", "visitsLabel": "Pertenece a la cita", diff --git a/messages/fr.json b/messages/fr.json index a1884c959..2c73fe8c1 100644 --- a/messages/fr.json +++ b/messages/fr.json @@ -6140,7 +6140,14 @@ "scopeNote": "Il peut uniquement ajouter de nouvelles mesures à votre propre dossier : il ne peut pas lire vos données, ni modifier ou supprimer quoi que ce soit, ni créer un autre jeton. Les mesures sont signalées comme provenant d'un appareil externe, ce qui vous permet de les reconnaître plus tard, et vous pouvez toujours les corriger.", "createFailed": "Le jeton n'a pas pu être créé. Veuillez réessayer." }, - "tokenMintMovedDescription": "Les jetons sont émis là où ils s'appliquent : ci-dessus pour les mesures, ou dans le point de terminaison API d'un médicament pour ce seul médicament.", + "documentsToken": { + "title": "Jetons de documents", + "description": "Créez un jeton pour que Paperless-ngx, Papra ou le script d'import ajoutent des documents à votre coffre.", + "detail": "Le jeton n'est affiché qu'une fois.", + "scopeNote": "Il peut uniquement ajouter de nouveaux documents à votre propre coffre. Il ne peut rien lister, ouvrir, télécharger, modifier ou supprimer, ni créer un autre jeton. Il apprend seulement si un fichier a été enregistré, pas comment vous l'avez classé.", + "createFailed": "Le jeton n'a pas pu être créé. Veuillez réessayer." + }, + "tokenMintMovedDescription": "Les jetons sont émis là où ils s'appliquent : ci-dessus pour les mesures et les documents, ou dans le point de terminaison API d'un médicament pour ce seul médicament.", "tokenNamePlaceholder": "Nom du jeton (par exemple Raccourci iPhone)", "tokenCreated": "Jeton créé — copiez-le maintenant ! Il ne sera plus affiché.", "tokenCopy": "Copier le jeton", @@ -9984,6 +9991,9 @@ "noConditions": "Lié à aucune affection", "previewUnavailable": "Pas d'aperçu intégré pour ce type de fichier", "uploadedMeta": "Ajouté le {date} · {size}", + "importedFrom": "Importé depuis {system}", + "importedFromWithId": "Importé depuis {system} · ID {id}", + "sourceOther": "un autre système", "share": "Partager", "labValuesLink": "Voir {count} valeurs de laboratoire", "visitsLabel": "Rattaché au rendez-vous", diff --git a/messages/it.json b/messages/it.json index 23b5b04d4..ded9178bc 100644 --- a/messages/it.json +++ b/messages/it.json @@ -6140,7 +6140,14 @@ "scopeNote": "Può soltanto aggiungere nuove misurazioni alla tua cartella: non può leggere i tuoi dati, modificare o eliminare nulla, né creare un altro token. Le misurazioni vengono contrassegnate come provenienti da un dispositivo esterno, così le riconosci in seguito, e puoi comunque correggerle.", "createFailed": "Non è stato possibile creare il token. Riprova." }, - "tokenMintMovedDescription": "I token vengono emessi dove si applicano: qui sopra per le misurazioni, oppure nell'endpoint API di un farmaco per quel singolo farmaco.", + "documentsToken": { + "title": "Token per documenti", + "description": "Crea un token perché Paperless-ngx, Papra o lo script di importazione aggiungano documenti al tuo archivio.", + "detail": "Il token viene mostrato una sola volta.", + "scopeNote": "Può soltanto aggiungere nuovi documenti al tuo archivio. Non può elencare, aprire, scaricare, modificare o eliminare nulla, né creare un altro token. Riceve solo l'informazione se un file è stato salvato, non come l'hai archiviato.", + "createFailed": "Non è stato possibile creare il token. Riprova." + }, + "tokenMintMovedDescription": "I token vengono emessi dove si applicano: qui sopra per le misurazioni e i documenti, oppure nell'endpoint API di un farmaco per quel singolo farmaco.", "tokenNamePlaceholder": "Nome del token (ad es. Comando rapido iPhone)", "tokenCreated": "Token creato — copialo ora! Non verrà mostrato di nuovo.", "tokenCopy": "Copia token", @@ -9984,6 +9991,9 @@ "noConditions": "Non collegato a una patologia", "previewUnavailable": "Nessuna anteprima integrata per questo tipo di file", "uploadedMeta": "Aggiunto il {date} · {size}", + "importedFrom": "Importato da {system}", + "importedFromWithId": "Importato da {system} · ID {id}", + "sourceOther": "un altro sistema", "share": "Condividi", "labValuesLink": "Vedi {count} valori di laboratorio", "visitsLabel": "Appartiene all'appuntamento", diff --git a/messages/ko.json b/messages/ko.json index fb5db9cc2..b63e830fe 100644 --- a/messages/ko.json +++ b/messages/ko.json @@ -6140,7 +6140,14 @@ "scopeNote": "본인 기록에 새 측정값을 추가하는 것만 할 수 있어요. 데이터를 읽거나, 무언가를 고치거나 지우거나, 다른 토큰을 만들 수는 없어요. 측정값은 외부 기기에서 온 것으로 표시되니 나중에 구분할 수 있고, 언제든 고칠 수 있어요.", "createFailed": "토큰을 만들지 못했어요. 다시 시도해 주세요." }, - "tokenMintMovedDescription": "토큰은 쓰이는 곳에서 발급돼요. 측정값은 위에서, 특정 약은 그 약의 API 엔드포인트에서 만들 수 있어요.", + "documentsToken": { + "title": "문서 토큰", + "description": "Paperless-ngx, Papra 또는 가져오기 스크립트가 문서함에 문서를 추가할 수 있도록 토큰을 만들어요.", + "detail": "토큰은 한 번만 표시돼요.", + "scopeNote": "본인 문서함에 새 문서를 추가하는 것만 할 수 있어요. 목록을 보거나, 열거나, 내려받거나, 고치거나 지울 수 없고, 다른 토큰을 만들 수도 없어요. 파일이 저장됐는지만 알 수 있고, 어떻게 정리했는지는 알 수 없어요.", + "createFailed": "토큰을 만들지 못했어요. 다시 시도해 주세요." + }, + "tokenMintMovedDescription": "토큰은 쓰이는 곳에서 발급돼요. 측정값과 문서는 위에서, 특정 약은 그 약의 API 엔드포인트에서 만들 수 있어요.", "tokenNamePlaceholder": "토큰 이름 (예: iPhone 단축어)", "tokenCreated": "토큰을 만들었어요 — 지금 복사하세요! 다시 보여주지 않아요.", "tokenCopy": "토큰 복사", @@ -9984,6 +9991,9 @@ "noConditions": "질환에 연결되지 않음", "previewUnavailable": "이 파일 형식은 미리보기를 지원하지 않아요", "uploadedMeta": "추가 {date} · {size}", + "importedFrom": "{system}에서 가져옴", + "importedFromWithId": "{system}에서 가져옴 · ID {id}", + "sourceOther": "다른 시스템", "share": "공유", "labValuesLink": "검사 수치 {count}개 보기", "visitsLabel": "연결된 진료", diff --git a/messages/pl.json b/messages/pl.json index cc0535130..3f4f9392c 100644 --- a/messages/pl.json +++ b/messages/pl.json @@ -6140,7 +6140,14 @@ "scopeNote": "Może wyłącznie dodawać nowe pomiary do Twojej dokumentacji — nie odczyta danych, niczego nie zmieni ani nie usunie i nie utworzy kolejnego tokenu. Pomiary są oznaczane jako pochodzące z urządzenia zewnętrznego, dzięki czemu później je rozpoznasz, i nadal możesz je poprawić.", "createFailed": "Nie udało się utworzyć tokenu. Spróbuj ponownie." }, - "tokenMintMovedDescription": "Tokeny wydajemy tam, gdzie działają: powyżej dla pomiarów albo w punkcie końcowym API danego leku dla tego jednego leku.", + "documentsToken": { + "title": "Tokeny dokumentów", + "description": "Utwórz token, aby Paperless-ngx, Papra lub skrypt importu mogły dodawać dokumenty do Twojego archiwum.", + "detail": "Token pokazujemy tylko raz.", + "scopeNote": "Może wyłącznie dodawać nowe dokumenty do Twojego archiwum. Niczego nie wyświetli, nie otworzy, nie pobierze, nie zmieni ani nie usunie i nie utworzy kolejnego tokenu. Dowiaduje się tylko, czy plik został zapisany, a nie to, jak został opisany.", + "createFailed": "Nie udało się utworzyć tokenu. Spróbuj ponownie." + }, + "tokenMintMovedDescription": "Tokeny wydajemy tam, gdzie działają: powyżej dla pomiarów i dokumentów albo w punkcie końcowym API danego leku dla tego jednego leku.", "tokenNamePlaceholder": "Nazwa tokena (np. Skrót iPhone)", "tokenCreated": "Token utworzony — skopiuj go teraz! Nie zostanie pokazany ponownie.", "tokenCopy": "Kopiuj token", @@ -9984,6 +9991,9 @@ "noConditions": "Niepowiązany ze schorzeniem", "previewUnavailable": "Brak podglądu dla tego typu pliku", "uploadedMeta": "Dodano {date} · {size}", + "importedFrom": "Zaimportowano z: {system}", + "importedFromWithId": "Zaimportowano z: {system} · ID {id}", + "sourceOther": "inny system", "share": "Udostępnij", "labValuesLink": "Zobacz {count} wyników laboratoryjnych", "visitsLabel": "Należy do wizyty", diff --git a/prisma/migrations/0353_document_import_source/migration.sql b/prisma/migrations/0353_document_import_source/migration.sql new file mode 100644 index 000000000..65c899381 --- /dev/null +++ b/prisma/migrations/0353_document_import_source/migration.sql @@ -0,0 +1,31 @@ +-- Document import (#1038). Two optional provenance columns on the vault, a +-- per-user unique source key that covers tombstoned rows too, and a small +-- ledger the purge fills so a re-run importer cannot bring back a deleted +-- document after its tombstone is gone. Existing rows keep NULL and are not +-- touched by the index. +ALTER TABLE "inbound_documents" + ADD COLUMN "source_system" VARCHAR(16), + ADD COLUMN "source_id" VARCHAR(128); + +-- Partial unique index (documented in schema.prisma; Prisma cannot express a +-- partial index). No `deleted_at` predicate on purpose: a tombstoned document +-- still owns its source key. +CREATE UNIQUE INDEX "inbound_documents_user_source_key" + ON "inbound_documents" ("user_id", "source_system", "source_id") + WHERE "source_id" IS NOT NULL; + +CREATE TABLE "document_import_keys" ( + "id" TEXT NOT NULL, + "user_id" TEXT NOT NULL, + "source_system" VARCHAR(16) NOT NULL, + "source_id" VARCHAR(128) NOT NULL, + "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + CONSTRAINT "document_import_keys_pkey" PRIMARY KEY ("id") +); + +CREATE UNIQUE INDEX "document_import_keys_user_id_source_system_source_id_key" + ON "document_import_keys" ("user_id", "source_system", "source_id"); + +ALTER TABLE "document_import_keys" + ADD CONSTRAINT "document_import_keys_user_id_fkey" + FOREIGN KEY ("user_id") REFERENCES "users"("id") ON DELETE CASCADE ON UPDATE CASCADE; diff --git a/prisma/schema.prisma b/prisma/schema.prisma index 9cff57b4e..202846026 100644 --- a/prisma/schema.prisma +++ b/prisma/schema.prisma @@ -922,6 +922,9 @@ model User { // (ExtractedFact) and only reach the structured stores on explicit // approval. Opt-in module, off by default. inboundDocuments InboundDocument[] + // v1.39.2 — source keys of imported documents that the tombstone purge + // removed, so a re-run of an importer cannot bring a deleted document back. + documentImportKeys DocumentImportKey[] extractedFacts ExtractedFact[] documentConditionLinks DocumentConditionLink[] documentContentIndexes DocumentContentIndex[] @@ -7938,6 +7941,22 @@ model InboundDocument { /// from "tried and here is why it failed" instead of showing a bare /// not-indexed state for both. lastIndexOutcome String? @map("last_index_outcome") + /// v1.39.2 (#1038) — where an imported document came from: a closed string + /// enum (`DOCUMENT_SOURCE_SYSTEMS` in + /// `src/lib/validations/inbound-documents.ts`): PAPERLESS | PAPRA | OTHER. + /// Null for a document added in HealthLog itself. Sent by the import script + /// and the Paperless workflow recipe; shown on the detail sheet. + sourceSystem String? @map("source_system") @db.VarChar(16) + /// v1.39.2 (#1038) — the document's id in `sourceSystem`. Together with + /// `userId` + `sourceSystem` it is unique across live AND tombstoned rows + /// (raw SQL in migration 0353, not expressible in Prisma): + /// CREATE UNIQUE INDEX inbound_documents_user_source_key + /// ON inbound_documents (user_id, source_system, source_id) + /// WHERE source_id IS NOT NULL; + /// Including tombstones is the point: re-running an importer must not bring + /// back a document the person deleted. The purge carries the key over to + /// `DocumentImportKey` when it removes the row. + sourceId String? @map("source_id") @db.VarChar(128) facts ExtractedFact[] conditionLinks DocumentConditionLink[] @@ -7984,6 +8003,26 @@ model InboundDocument { @@map("inbound_documents") } +/// v1.39.2 (#1038) — the source key of an imported document that the daily +/// tombstone purge has physically removed. While the tombstone exists the key +/// lives on the document row itself; once the purge takes the row, this is the +/// only memory that the person deleted it, so a re-run of the importer answers +/// "deleted" instead of storing the document again. Written by the purge only; +/// read by the upload route's source-key check. Holds no document content. +model DocumentImportKey { + id String @id @default(cuid()) + userId String @map("user_id") + /// Same closed vocabulary as `InboundDocument.sourceSystem`. + sourceSystem String @map("source_system") @db.VarChar(16) + sourceId String @map("source_id") @db.VarChar(128) + createdAt DateTime @default(now()) @map("created_at") + + user User @relation(fields: [userId], references: [id], onDelete: Cascade) + + @@unique([userId, sourceSystem, sourceId]) + @@map("document_import_keys") +} + model ExtractedFact { id String @id @default(cuid()) documentId String @map("document_id") diff --git a/scripts/env-manifest.json b/scripts/env-manifest.json index 83b7df229..8a87ecc59 100644 --- a/scripts/env-manifest.json +++ b/scripts/env-manifest.json @@ -138,6 +138,10 @@ "name": "DOCUMENT_AI_LIMIT_PER_HOUR", "purpose": "Per-user hourly ceiling for the document-AI actions (read/index, extract, suggest, summarise share one bucket). Default 6, clamped to 1-1000." }, + { + "name": "DOCUMENT_UPLOAD_LIMIT_PER_HOUR", + "purpose": "Hourly upload ceiling per document-upload token (Paperless workflow, import script). Default 120, clamped to 1-1000. Web and phone uploads keep their own bucket." + }, { "name": "LABS_OCR_LIMIT_PER_HOUR", "purpose": "Per-user hourly ceiling for lab-report scans (photo/PDF and browser-OCR text share one bucket). Default 6, clamped to 1-1000." diff --git a/src/__tests__/bearer-scope-enforcement-guard.test.ts b/src/__tests__/bearer-scope-enforcement-guard.test.ts index dcc05ca09..2e28046f5 100644 --- a/src/__tests__/bearer-scope-enforcement-guard.test.ts +++ b/src/__tests__/bearer-scope-enforcement-guard.test.ts @@ -149,6 +149,9 @@ describe("T3 — the mint sites are frozen", () => { // body cannot express a scope. Reaches two write routes on the holder's // own record and nothing else, the mint included. "app/api/tokens/measurements/route.ts": "[MEASUREMENTS_WRITE_SCOPE]", + // Document upload from another system (#1038). Same shape as the + // measurement mint: a literal, one scope, one route that accepts it. + "app/api/tokens/documents/route.ts": "[DOCUMENTS_WRITE_SCOPE]", // MCP, audience-bound to `/mcp`. `health:write` requires explicit consent. "app/api/mcp/tokens/route.ts": "SCOPE_HEALTH_READ / SCOPE_HEALTH_WRITE", "app/api/mcp/oauth/token/route.ts": @@ -175,6 +178,7 @@ describe("T3 — the mint sites are frozen", () => { const userFacing = [ "app/api/medications/[id]/api-endpoint/route.ts", "app/api/tokens/measurements/route.ts", + "app/api/tokens/documents/route.ts", "app/api/mcp/tokens/route.ts", "app/api/mcp/oauth/token/route.ts", ]; @@ -226,7 +230,11 @@ describe("T4 — declared scopes are exported constants, never string literals", // `scope` option, a route can widen itself without the word `requireAuth` // appearing anywhere in it. A vocabulary that watched only the first form // would have gone on passing while the surface it describes grew. - const DECLARED_SCOPES = ["FHIR_READ_SCOPE", "MEASUREMENTS_WRITE_SCOPE"]; + const DECLARED_SCOPES = [ + "DOCUMENTS_WRITE_SCOPE", + "FHIR_READ_SCOPE", + "MEASUREMENTS_WRITE_SCOPE", + ]; const used = new Set(); for (const rel of sourceFiles()) { const src = read(rel); @@ -245,6 +253,24 @@ describe("T4 — declared scopes are exported constants, never string literals", }); }); +describe("T4b — `documents:write` opens exactly one door", () => { + it("only the vault upload route declares the scope", () => { + // The scope admits a narrow token wherever it is named, so the set of + // files naming it IS its reach. The upload, and nothing else: not the + // list or detail beside it, not the original or thumbnail, not bulk, not + // the AI legs. A second route here is a widening and belongs in review. + const declaring = filesMatching( + /requireAuth\(\s*DOCUMENTS_WRITE_SCOPE\s*\)|requireRecordAuth\([^)]*\bscope:\s*DOCUMENTS_WRITE_SCOPE\b/, + ); + expect(declaring).toEqual(["app/api/documents/inbound/route.ts"]); + // And within that file, once: the GET beside the POST must not name it. + const src = read("app/api/documents/inbound/route.ts"); + expect( + src.match(/requireAuth\(\s*DOCUMENTS_WRITE_SCOPE\s*\)/g), + ).toHaveLength(1); + }); +}); + describe("T5 — a delegable route naming a Bearer scope is frozen", () => { /** * Routes that admit a narrow token on a DELEGABLE surface, and the scope each diff --git a/src/app/api/documents/inbound/__tests__/route.test.ts b/src/app/api/documents/inbound/__tests__/route.test.ts index 265bc6e6d..b144b1420 100644 --- a/src/app/api/documents/inbound/__tests__/route.test.ts +++ b/src/app/api/documents/inbound/__tests__/route.test.ts @@ -1,4 +1,4 @@ -import { beforeEach, describe, expect, it, vi } from "vitest"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; /** * Document vault: store-only upload + browsable list. @@ -50,6 +50,9 @@ vi.mock("@/lib/db", () => { findMany: vi.fn(), findFirst: vi.fn(), }, + documentImportKey: { + findUnique: vi.fn(), + }, extractedFact: { groupBy: vi.fn(), }, @@ -114,6 +117,12 @@ vi.mock("@/lib/rate-limit", () => ({ })); vi.mock("@/lib/auth/session", () => ({ getSession: vi.fn() })); +// The Bearer path: the real `requireAuth` runs, and only the token lookup is +// stubbed, so the scope decision under test is the production one. +vi.mock("@/lib/auth/bearer", async (importOriginal) => { + const actual = await importOriginal(); + return { ...actual, resolveBearerToken: vi.fn() }; +}); vi.mock("@/lib/auth/audit", () => ({ auditLog: vi.fn().mockResolvedValue(undefined), })); @@ -135,6 +144,11 @@ import { createHash } from "node:crypto"; import { POST, GET } from "../route"; import { prisma } from "@/lib/db"; import { getSession } from "@/lib/auth/session"; +import { headers } from "next/headers"; +import { resolveBearerToken, BearerAuthError } from "@/lib/auth/bearer"; +import { checkRateLimit } from "@/lib/rate-limit"; +import { enqueueDocumentThumbnail } from "@/lib/jobs/document-thumbnail"; +import { DOCUMENTS_WRITE_SCOPE } from "@/lib/documents/scopes"; import { requireModuleEnabled } from "@/lib/modules/gate"; import { enqueueDocumentIndex } from "@/lib/jobs/document-index"; import { enqueueDocumentSummary } from "@/lib/jobs/document-summary"; @@ -219,6 +233,9 @@ beforeEach(() => { [] as never, ); vi.mocked(prisma.inboundDocument.findFirst).mockResolvedValue(null as never); + vi.mocked(prisma.documentImportKey.findUnique).mockResolvedValue( + null as never, + ); // No settings row / no user override → policy defaults (25 MiB / 1 GiB). vi.mocked(prisma.appSettings.findUnique).mockResolvedValue(null as never); vi.mocked(prisma.user.findUnique).mockResolvedValue(null as never); @@ -581,3 +598,216 @@ describe("POST /api/documents/inbound — AI work on upload follows documentAi", expect(enqueueDocumentSummary).not.toHaveBeenCalled(); }); }); + +/** + * v1.39.2 (#1038) — the narrow `documents:write` token, and the import fields + * any caller may send. The Bearer is resolved by the real `requireAuth`; only + * the token lookup is stubbed. + */ +describe("POST /api/documents/inbound — document import", () => { + const TOKEN = "hlk_" + "d".repeat(64); + + function armToken(permissions: string[]) { + vi.mocked(getSession).mockResolvedValue(null as never); + vi.mocked(headers).mockResolvedValue({ + get: (name: string) => + name.toLowerCase() === "authorization" ? `Bearer ${TOKEN}` : null, + } as never); + vi.mocked(resolveBearerToken).mockImplementation(async (_raw, req) => { + const admitted = + permissions.includes("*") || + (req.kind === "scope" && permissions.includes(req.scope)); + if (!admitted) { + throw new BearerAuthError( + 403, + "insufficient_permissions", + "user-1", + "tok-1", + ); + } + return { + user: { id: "user-1", username: "tester", role: "USER" }, + tokenId: "tok-1", + expiresAt: new Date(Date.now() + 3_600_000), + permissions, + } as never; + }); + } + + beforeEach(() => { + vi.mocked(checkRateLimit).mockResolvedValue({ + allowed: true, + remaining: 59, + resetAt: Date.now() + 3_600_000, + } as never); + }); + + afterEach(() => { + vi.mocked(headers).mockResolvedValue({ get: () => null } as never); + }); + + it("admits the scoped token and answers with a receipt, not the row", async () => { + armToken([DOCUMENTS_WRITE_SCOPE]); + txCreate.mockResolvedValue( + docRow({ title: "Befund", filename: "befund.png" }) as never, + ); + + const res = await post(mkUpload({ title: "Befund" })); + expect(res.status).toBe(201); + const body = await res.json(); + // Exactly the receipt: no title, filename, kind, links or counts. + expect(body.data).toEqual({ id: "doc-1", duplicate: false }); + }); + + it("gives the scoped token its own bucket, keyed on the token", async () => { + armToken([DOCUMENTS_WRITE_SCOPE]); + await post(mkUpload()); + const key = vi.mocked(checkRateLimit).mock.calls[0]![0]; + expect(key).toBe("documents-upload:token:tok-1"); + // Default 120/hour for a token; the cookie bucket stays at 60. + expect(vi.mocked(checkRateLimit).mock.calls[0]![1]).toBe(120); + }); + + it("keeps the cookie bucket for a session", async () => { + await post(mkUpload()); + expect(vi.mocked(checkRateLimit).mock.calls[0]![0]).toBe( + "documents-upload:user-1", + ); + expect(vi.mocked(checkRateLimit).mock.calls[0]![1]).toBe(60); + }); + + it("429s the scoped token past its bucket with Retry-After", async () => { + armToken([DOCUMENTS_WRITE_SCOPE]); + vi.mocked(checkRateLimit).mockResolvedValue({ + allowed: false, + remaining: 0, + limit: 120, + resetAt: Date.now() + 90_000, + } as never); + const res = await post(mkUpload()); + expect(res.status).toBe(429); + expect((await res.json()).meta.errorCode).toBe( + "documents.inbound.rateLimited", + ); + expect(txCreate).not.toHaveBeenCalled(); + }); + + it("refuses a token carrying some other narrow scope", async () => { + armToken(["measurements:write"]); + const res = await post(mkUpload()); + expect(res.status).toBe(403); + expect(txCreate).not.toHaveBeenCalled(); + }); + + it("answers a scoped content duplicate with the receipt", async () => { + armToken([DOCUMENTS_WRITE_SCOPE]); + vi.mocked(prisma.inboundDocument.findFirst).mockResolvedValue( + docRow({ id: "doc-existing", title: "Private title" }) as never, + ); + const res = await post(mkUpload()); + expect(res.status).toBe(200); + const body = await res.json(); + expect(body.data).toEqual({ id: "doc-existing", duplicate: true }); + expect(body.meta.duplicate).toBe(true); + }); + + it("stores the source key on a fresh import", async () => { + armToken([DOCUMENTS_WRITE_SCOPE]); + await post(mkUpload({ sourceSystem: "PAPERLESS", sourceId: "412" })); + const arg = txCreate.mock.calls[0]![0]!; + expect(arg.data.sourceSystem).toBe("PAPERLESS"); + expect(arg.data.sourceId).toBe("412"); + }); + + it("answers a live source-key match as a duplicate without reading bytes", async () => { + vi.mocked(prisma.inboundDocument.findFirst).mockResolvedValueOnce( + docRow({ + id: "doc-src", + sourceSystem: "PAPRA", + sourceId: "doc_1", + }) as never, + ); + const res = await post( + mkUpload({ sourceSystem: "PAPRA", sourceId: "doc_1" }), + ); + expect(res.status).toBe(200); + const body = await res.json(); + expect(body.meta.duplicate).toBe(true); + expect(body.data.id).toBe("doc-src"); + // The lookup spans tombstones: no deletedAt predicate. + const where = vi.mocked(prisma.inboundDocument.findFirst).mock.calls[0]![0]! + .where!; + expect(where).toEqual({ + userId: "user-1", + sourceSystem: "PAPRA", + sourceId: "doc_1", + }); + expect(txCreate).not.toHaveBeenCalled(); + }); + + it("refuses to bring back a deleted import: 200 deleted, no new row", async () => { + vi.mocked(prisma.inboundDocument.findFirst).mockResolvedValueOnce( + docRow({ + id: "doc-gone", + sourceSystem: "PAPERLESS", + sourceId: "7", + deletedAt: new Date("2026-09-01T00:00:00.000Z"), + }) as never, + ); + const res = await post( + mkUpload({ sourceSystem: "PAPERLESS", sourceId: "7" }), + ); + expect(res.status).toBe(200); + const body = await res.json(); + expect(body.data).toEqual({ + id: "doc-gone", + duplicate: true, + deleted: true, + }); + expect(body.meta).toEqual({ duplicate: true, deleted: true }); + expect(txCreate).not.toHaveBeenCalled(); + expect(enqueueDocumentIndex).not.toHaveBeenCalled(); + }); + + it("remembers a purged import through the ledger", async () => { + vi.mocked(prisma.documentImportKey.findUnique).mockResolvedValue({ + id: "key-1", + } as never); + const res = await post( + mkUpload({ sourceSystem: "PAPERLESS", sourceId: "7" }), + ); + expect(res.status).toBe(200); + const body = await res.json(); + expect(body.data).toEqual({ id: null, duplicate: true, deleted: true }); + expect(txCreate).not.toHaveBeenCalled(); + }); + + it("422s a sourceId without a sourceSystem", async () => { + const res = await post(mkUpload({ sourceId: "7" })); + expect(res.status).toBe(422); + expect(txCreate).not.toHaveBeenCalled(); + }); + + it("422s a sourceSystem outside the closed set", async () => { + const res = await post( + mkUpload({ sourceSystem: "DROPBOX", sourceId: "7" }), + ); + expect(res.status).toBe(422); + }); + + it("aiRead=defer indexes locally and skips the summary", async () => { + const res = await post(mkUpload({ aiRead: "defer" })); + expect(res.status).toBe(201); + expect(enqueueDocumentIndex).toHaveBeenCalledWith("user-1", "doc-1", { + localOnly: true, + }); + expect(enqueueDocumentThumbnail).toHaveBeenCalledWith("user-1", "doc-1"); + expect(enqueueDocumentSummary).not.toHaveBeenCalled(); + }); + + it("without aiRead the upload keeps today's AI behaviour", async () => { + await post(mkUpload()); + expect(enqueueDocumentIndex).toHaveBeenCalledWith("user-1", "doc-1"); + expect(enqueueDocumentSummary).toHaveBeenCalledWith("user-1", "doc-1"); + }); +}); diff --git a/src/app/api/documents/inbound/route.ts b/src/app/api/documents/inbound/route.ts index a10dcc3ba..e38c5d284 100644 --- a/src/app/api/documents/inbound/route.ts +++ b/src/app/api/documents/inbound/route.ts @@ -15,6 +15,15 @@ * (a same-user re-upload returns the existing live row, never a second copy), * an honoured `Idempotency-Key`, and optional `episodeIds[]` pre-linking. * + * v1.39.2 (#1038) — POST also admits a narrow `documents:write` token, the + * door another document system (a Paperless workflow, the import script) + * pushes through. Such a caller gets its own rate bucket, and a receipt + * (`{ id, duplicate }`) instead of the stored row: a credential that can only + * add files learns that a file exists, never how its owner filed it. Optional + * `sourceSystem` / `sourceId` fields key an import so a re-send is a + * duplicate, including after the person deleted the document; `aiRead=defer` + * holds back automatic AI reading for the upload. + * * GET lists the caller's documents with title/filename search, kind / * episode / year / date-range filters, sort, and keyset pagination — and * NEVER selects the encrypted blob column (`omit: { contentEncrypted: true }`). @@ -29,6 +38,7 @@ import { NextResponse } from "next/server"; import { apiHandler, + isScopedCredential, requireAuth, requireRecordAuth, type AuthContext, @@ -58,10 +68,12 @@ import { import { enqueueDocumentIndex } from "@/lib/jobs/document-index"; import { enqueueDocumentSummary } from "@/lib/jobs/document-summary"; import { enqueueDocumentThumbnail } from "@/lib/jobs/document-thumbnail"; +import { DOCUMENTS_WRITE_SCOPE } from "@/lib/documents/scopes"; import { acquireDocumentUploadSlot, detectDocumentType, resolveDocumentLimits, + resolveDocumentUploadLimitPerHour, } from "@/lib/documents/upload-policy"; import { withIdempotency } from "@/lib/idempotency"; import { BodyTooLargeError, readBoundedBody } from "@/lib/labs/ocr-upload"; @@ -74,6 +86,7 @@ import { documentCreateSchema, documentListQuerySchema, toContentIndexSource, + type DocumentSourceSystemValue, } from "@/lib/validations/inbound-documents"; import type { Prisma } from "@/generated/prisma/client"; @@ -116,15 +129,93 @@ class QuotaExceededError extends Error { } } +/** What a single upload asked for, carried into the annotations. */ +interface UploadContext { + userId: string; + /** A narrow `documents:write` token, not a session or a `["*"]` token. */ + scoped: boolean; + sourceSystem: DocumentSourceSystemValue | null; + aiDeferred: boolean; +} + +/** + * The receipt a scoped caller gets in place of the stored row: the id and + * whether the upload was a duplicate. A write-only credential holder who + * knows a file's bytes learns that it is stored, not its title, filename, + * links or fact counts. + */ +function receiptResponse( + id: string | null, + status: 200 | 201, + flags: { duplicate: boolean; deleted?: boolean }, +): NextResponse { + const data = { + id, + duplicate: flags.duplicate, + ...(flags.deleted ? { deleted: true } : {}), + }; + return NextResponse.json( + { + data, + error: null, + ...(flags.duplicate + ? { + meta: { + duplicate: true, + ...(flags.deleted ? { deleted: true } : {}), + }, + } + : {}), + }, + { status }, + ); +} + +/** + * v1.39.2 (#1038) — a re-send of an imported document the person deleted. + * 200 and no new row, for every caller: the deletion was a decision, and an + * importer re-running over the same source must not undo it. `id` is the + * tombstoned row while it exists and null once the purge took it; there is no + * row left to serialise either way, so the answer is the receipt. + */ +function deletedResponse(ctx: UploadContext, id: string | null): NextResponse { + annotate({ + action: { name: "documents.vault.upload" }, + meta: { + documentId: id, + duplicate: true, + deleted: true, + scoped: ctx.scoped, + sourceSystem: ctx.sourceSystem, + aiDeferred: ctx.aiDeferred, + }, + }); + return receiptResponse(id, 200, { duplicate: true, deleted: true }); +} + /** * §3.2 — a duplicate upload is NOT an error: return the existing live row * with `meta.duplicate: true` at the envelope level (the UI toasts "already - * stored" and highlights the row). + * stored" and highlights the row). A scoped caller gets the receipt instead. */ async function duplicateResponse( - userId: string, + ctx: UploadContext, existing: SerialisableDocument, ): Promise { + annotate({ + action: { name: "documents.vault.upload" }, + meta: { + documentId: existing.id, + duplicate: true, + scoped: ctx.scoped, + sourceSystem: ctx.sourceSystem, + aiDeferred: ctx.aiDeferred, + }, + }); + if (ctx.scoped) { + return receiptResponse(existing.id, 200, { duplicate: true }); + } + const userId = ctx.userId; const [links, visitLinks, groups] = await Promise.all([ loadConditionLinks(userId, [existing.id]), loadDocumentEncounterLinks(userId, [existing.id]), @@ -140,10 +231,6 @@ async function duplicateResponse( if (g.status !== "REJECTED") factCount += g._count._all; if (g.status === "PENDING") pendingCount += g._count._all; } - annotate({ - action: { name: "documents.vault.upload" }, - meta: { documentId: existing.id, duplicate: true }, - }); return NextResponse.json( { data: serialiseDocument( @@ -165,18 +252,31 @@ async function duplicateResponse( /** Process one admitted upload. The caller owns and releases its memory slot. */ async function processUpload( request: Request, - user: AuthContext["user"], + auth: AuthContext, ): Promise { + const { user } = auth; + const scoped = isScopedCredential(auth); + // Opt-in module gate — even a valid Bearer token is refused when the surface // is off (it ships dark; the user turns it on deliberately). const gate = await requireModuleEnabled(user.id, "inboundDocuments"); if (!gate.enabled) return gate.response; - const rl = await checkRateLimit( - `documents-upload:${user.id}`, - UPLOAD_LIMIT_PER_HOUR, - UPLOAD_WINDOW_MS, - ); + // A scoped token draws on its own bucket, keyed on the token (its id rides + // `session.id` on the Bearer path), so an import running overnight cannot + // use up the allowance the person's own uploads from the web or the phone + // draw on. The cookie / wildcard bucket is unchanged. + const rl = scoped + ? await checkRateLimit( + `documents-upload:token:${auth.session.id}`, + resolveDocumentUploadLimitPerHour(), + UPLOAD_WINDOW_MS, + ) + : await checkRateLimit( + `documents-upload:${user.id}`, + UPLOAD_LIMIT_PER_HOUR, + UPLOAD_WINDOW_MS, + ); if (!rl.allowed) { const response = apiError("Too many uploads. Try again later.", 429, { errorCode: "documents.inbound.rateLimited", @@ -241,6 +341,9 @@ async function processUpload( documentDate: formData.get("documentDate") ?? undefined, episodeIds: rawEpisodeIds.length > 0 ? rawEpisodeIds : undefined, encounterIds: rawEncounterIds.length > 0 ? rawEncounterIds : undefined, + sourceSystem: formData.get("sourceSystem") ?? undefined, + sourceId: formData.get("sourceId") ?? undefined, + aiRead: formData.get("aiRead") ?? undefined, }); if (!parsed.success) { return apiValidationError( @@ -253,6 +356,39 @@ async function processUpload( ); } + const sourceSystem = parsed.data.sourceSystem ?? null; + const sourceId = sourceSystem ? (parsed.data.sourceId ?? null) : null; + const ctx: UploadContext = { + userId: user.id, + scoped, + sourceSystem, + aiDeferred: parsed.data.aiRead === "defer", + }; + + // A source key answers before the bytes are looked at: an import re-sending + // what it sent before is a duplicate, and one re-sending a document the + // person deleted is refused a second copy — live and tombstoned rows both + // count (the unique index has no `deleted_at` predicate), and past the purge + // the ledger remembers. + if (sourceSystem && sourceId) { + const keyed = await findBySourceKey(user.id, sourceSystem, sourceId); + if (keyed) + return keyed.deletedAt + ? deletedResponse(ctx, keyed.id) + : duplicateResponse(ctx, keyed); + const purged = await prisma.documentImportKey.findUnique({ + where: { + userId_sourceSystem_sourceId: { + userId: user.id, + sourceSystem, + sourceId, + }, + }, + select: { id: true }, + }); + if (purged) return deletedResponse(ctx, null); + } + let buffer: Buffer; try { buffer = Buffer.from(await file.arrayBuffer()); @@ -313,7 +449,7 @@ async function processUpload( omit: { contentEncrypted: true }, }); if (existing) { - return duplicateResponse(user.id, existing); + return duplicateResponse(ctx, existing); } // Quota gate + insert + pre-links in ONE transaction. Usage counts every @@ -369,6 +505,8 @@ async function processUpload( documentDate: parsed.data.documentDate ? isoDateToUtc(parsed.data.documentDate) : new Date(), + sourceSystem, + sourceId, }, omit: { contentEncrypted: true }, }); @@ -403,13 +541,22 @@ async function processUpload( }); } if (isP2002(err)) { - // A racing upload of the same bytes won the partial unique index. - // Surface the winner as the duplicate — same outcome as the fast path. + // A racing upload won one of the two partial unique indexes — the same + // source key, or the same bytes. Surface the winner exactly as the fast + // paths above would have. + if (sourceSystem && sourceId) { + const keyed = await findBySourceKey(user.id, sourceSystem, sourceId); + if (keyed) { + return keyed.deletedAt + ? deletedResponse(ctx, keyed.id) + : duplicateResponse(ctx, keyed); + } + } const winner = await prisma.inboundDocument.findFirst({ where: { userId: user.id, contentSha256, deletedAt: null }, omit: { contentEncrypted: true }, }); - if (winner) return duplicateResponse(user.id, winner); + if (winner) return duplicateResponse(ctx, winner); } throw err; } @@ -417,7 +564,12 @@ async function processUpload( await auditLog("documents.inbound.store", { userId: user.id, ipAddress: getClientIp(request), - details: { documentId: document.id, mime: detected.mimeType }, + details: { + documentId: document.id, + mime: detected.mimeType, + ...(scoped ? { scoped: true } : {}), + ...(sourceSystem ? { sourceSystem } : {}), + }, }); annotate({ @@ -428,6 +580,9 @@ async function processUpload( servingClass: detected.servingClass, linked: episodeIds.length, linkedVisits: encounterIds.length, + scoped, + sourceSystem, + aiDeferred: ctx.aiDeferred, }, }); @@ -437,7 +592,15 @@ async function processUpload( // enqueue is not awaited and swallows its own errors (a missing boss or a // transient send failure is a silent no-op). Only fresh inserts enqueue — a // duplicate upload returns early above and never reaches here. - void enqueueDocumentIndex(user.id, document.id); + // + // `aiRead=defer` narrows it to the local text layer: an import of a whole + // archive must not turn into one provider call per file. The person reads + // them later, deliberately, from the document itself. + if (ctx.aiDeferred) { + void enqueueDocumentIndex(user.id, document.id, { localOnly: true }); + } else { + void enqueueDocumentIndex(user.id, document.id); + } // Render a preview thumbnail in the background too (pure local compute — no // egress). Same fire-and-forget contract: the upload never blocks on or fails @@ -453,7 +616,12 @@ async function processUpload( // provider-free text-layer path. The persisted summary shows on the detail // view. Same fire-and-forget contract: the upload never blocks on or fails // because of it. Only fresh inserts reach here (a duplicate returns early). - const documentAi = await getAiCapability("documentAi"); + // + // A deferred upload skips it outright and keeps `summaryState: NONE`, so + // the detail sheet offers "Generate summary" rather than a pending state. + const documentAi = ctx.aiDeferred + ? { available: false as const, reason: "deferred" } + : await getAiCapability("documentAi"); if (documentAi.available) { void enqueueDocumentSummary(user.id, document.id); } else { @@ -463,6 +631,8 @@ async function processUpload( }); } + if (scoped) return receiptResponse(document.id, 201, { duplicate: false }); + const [links, visitLinks] = await Promise.all([ loadConditionLinks(user.id, [document.id]), loadDocumentEncounterLinks(user.id, [document.id]), @@ -481,9 +651,28 @@ async function processUpload( ); } +/** + * Find a document by its import source key, live or tombstoned. The blob + * column is never selected. + */ +function findBySourceKey( + userId: string, + sourceSystem: DocumentSourceSystemValue, + sourceId: string, +): Promise { + return prisma.inboundDocument.findFirst({ + where: { userId, sourceSystem, sourceId }, + omit: { contentEncrypted: true }, + }); +} + /** Authenticate and reserve memory capacity before any request-body read. */ async function postUpload(request: Request): Promise { - const { user } = await requireAuth(); + // The one route that names `documents:write`. Every other vault leg — + // list, detail, original, thumbnail, bulk, AI — declares no scope and so + // refuses the token (`bearer-scope-enforcement-guard.test.ts`). + const auth = await requireAuth(DOCUMENTS_WRITE_SCOPE); + const { user } = auth; const releaseUploadSlot = acquireDocumentUploadSlot(user.id); if (!releaseUploadSlot) { const response = apiError( @@ -499,7 +688,7 @@ async function postUpload(request: Request): Promise { } try { - return await processUpload(request, user); + return await processUpload(request, auth); } finally { releaseUploadSlot(); } @@ -517,7 +706,7 @@ export const POST = apiHandler(withIdempotency<[Request]>(postUpload)); * letter is health data belonging to the record, and reading it is the whole * point of handing somebody the record. * - * The POST above is NOT delegable and keeps `requireAuth()`. Uploading is not + * The POST above is NOT delegable and declares its own scope. Uploading is not * a verb the grant admits, and the module-level split matters here: the * resolver escalates any non-safe method to `"write"`, so the two arms have to * declare separately rather than share one line. diff --git a/src/app/api/tokens/documents/__tests__/route.test.ts b/src/app/api/tokens/documents/__tests__/route.test.ts new file mode 100644 index 000000000..d6cec1431 --- /dev/null +++ b/src/app/api/tokens/documents/__tests__/route.test.ts @@ -0,0 +1,220 @@ +/** + * The document-upload mint (#1038). + * + * The assertion that matters most is not the 201 — it is the exact + * `permissions` array handed to `issueApiToken`. That helper defaults to + * `["*"]` when the property is absent, so an edit that drops one line turns a + * user-facing endpoint into a cookie-equivalent credential factory, and a test + * that only checked the status code would go on passing. The structural guard + * catches a literal `permissions: ["*"` and would not catch the omission, so + * this is where that hole is closed. + */ +import { describe, it, expect, vi, beforeEach } from "vitest"; + +vi.mock("@/lib/api-handler", () => ({ + apiHandler: (fn: unknown) => fn, + requireCookieAuth: vi.fn(), +})); + +vi.mock("@/lib/auth/issue-token", () => ({ + issueApiToken: vi.fn(), +})); + +vi.mock("@/lib/app-settings", () => ({ + isApiGloballyEnabled: vi.fn(), +})); + +vi.mock("@/lib/rate-limit", () => ({ + checkRateLimit: vi.fn(), +})); + +vi.mock("@/lib/auth/audit", () => ({ + auditLog: vi.fn().mockResolvedValue(undefined), +})); + +vi.mock("@/lib/logging/context", () => ({ + annotate: vi.fn(), + getEvent: vi.fn(() => null), +})); + +vi.mock("@/lib/db", () => ({ + prisma: { apiToken: { count: vi.fn() } }, +})); + +import { POST } from "../route"; +import { requireCookieAuth } from "@/lib/api-handler"; +import { issueApiToken } from "@/lib/auth/issue-token"; +import { isApiGloballyEnabled } from "@/lib/app-settings"; +import { checkRateLimit } from "@/lib/rate-limit"; +import { auditLog } from "@/lib/auth/audit"; +import { prisma } from "@/lib/db"; +import { DOCUMENTS_WRITE_SCOPE } from "@/lib/documents/scopes"; + +const USER = { id: "user-1", role: "USER" as const }; +const EXPIRES = new Date("2027-08-31T00:00:00.000Z"); + +function req(body: unknown): Request { + return new Request("https://health.example/api/tokens/documents", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify(body), + }); +} + +beforeEach(() => { + vi.resetAllMocks(); + vi.mocked(requireCookieAuth).mockResolvedValue({ user: USER } as never); + vi.mocked(isApiGloballyEnabled).mockResolvedValue(true); + vi.mocked(checkRateLimit).mockResolvedValue({ allowed: true } as never); + vi.mocked(auditLog).mockResolvedValue(undefined as never); + // Below the ceiling by default; the cases that care set their own count. + vi.mocked(prisma.apiToken.count).mockResolvedValue(0 as never); + vi.mocked(issueApiToken).mockResolvedValue({ + token: "hlk_" + "a".repeat(64), + expiresAt: EXPIRES, + tokenId: "token-1", + name: "Paperless workflow", + } as never); +}); + +describe("what it mints", () => { + it("names exactly the upload scope, and no wildcard", async () => { + const res = await POST(req({ name: "Paperless workflow" }) as never); + expect(res.status).toBe(201); + + const opts = vi.mocked(issueApiToken).mock.calls[0][0]; + // Exact equality, not `toContain`: an extra scope is a widening, and this + // is the assertion standing between the endpoint and one. + expect(opts.permissions).toEqual([DOCUMENTS_WRITE_SCOPE]); + expect(opts.userId).toBe(USER.id); + }); + + it("returns the raw token once, with its name and expiry", async () => { + const res = await POST(req({ name: "Paperless workflow" }) as never); + const json = await res.json(); + expect(json.data.token).toMatch(/^hlk_[0-9a-f]{64}$/); + expect(json.data.name).toBe("Paperless workflow"); + expect(json.data.expiresAt).toBeDefined(); + }); + + it("expires by default rather than living forever", async () => { + await POST(req({ name: "Import script" }) as never); + const opts = vi.mocked(issueApiToken).mock.calls[0][0]; + expect(opts.expiresInDays).toBe(365); + }); + + it("honours an explicit lifetime", async () => { + await POST(req({ name: "Import script", expiresInDays: 30 }) as never); + expect(vi.mocked(issueApiToken).mock.calls[0][0].expiresInDays).toBe(30); + }); + + it("audits the mint with the token id and the scope", async () => { + await POST(req({ name: "Import script" }) as never); + expect(auditLog).toHaveBeenCalledWith( + "tokens.documents.create", + expect.objectContaining({ + userId: USER.id, + details: expect.objectContaining({ + tokenId: "token-1", + scope: DOCUMENTS_WRITE_SCOPE, + }), + }), + ); + }); + + it("counts only live tokens carrying this scope", async () => { + // The whole shape of the query, asserted rather than assumed. Counting + // every row would let the `["*"]` tokens a login mints consume the budget + // and refuse a mint for a reason the person cannot see; counting revoked + // or expired rows would mean revoking never frees a slot. + await POST(req({ name: "Import script" }) as never); + + const where = vi.mocked(prisma.apiToken.count).mock.calls[0][0]?.where; + expect(where).toMatchObject({ + userId: USER.id, + revoked: false, + permissions: { has: DOCUMENTS_WRITE_SCOPE }, + }); + expect(where?.OR).toEqual([ + { expiresAt: null }, + { expiresAt: { gt: expect.any(Date) } }, + ]); + }); +}); + +describe("when it refuses", () => { + it("403s while the operator has the API switched off", async () => { + vi.mocked(isApiGloballyEnabled).mockResolvedValue(false); + const res = await POST(req({ name: "Import script" }) as never); + expect(res.status).toBe(403); + expect(issueApiToken).not.toHaveBeenCalled(); + }); + + it("429s past the mint rate limit, without minting", async () => { + vi.mocked(checkRateLimit).mockResolvedValue({ allowed: false } as never); + const res = await POST(req({ name: "Import script" }) as never); + expect(res.status).toBe(429); + expect(issueApiToken).not.toHaveBeenCalled(); + }); + + it("422s on a missing name", async () => { + const res = await POST(req({}) as never); + expect(res.status).toBe(422); + expect(issueApiToken).not.toHaveBeenCalled(); + }); + + it("422s on a lifetime outside the permitted band", async () => { + const res = await POST( + req({ name: "Import script", expiresInDays: 4000 }) as never, + ); + expect(res.status).toBe(422); + expect(issueApiToken).not.toHaveBeenCalled(); + }); + + it("409s at the live-token ceiling, without minting", async () => { + vi.mocked(prisma.apiToken.count).mockResolvedValue(10 as never); + const res = await POST(req({ name: "Import script" }) as never); + + // 409, not 429: a conflict with the account's current state, whose remedy + // is revoking rather than waiting. A 429 would say the opposite. + expect(res.status).toBe(409); + expect((await res.json()).meta?.errorCode).toBe( + "tokens.documents.ceiling_reached", + ); + expect(issueApiToken).not.toHaveBeenCalled(); + }); + + it("still mints on the last free slot", async () => { + // The off-by-one that would make the ceiling nine. Worth pinning, because + // `>=` versus `>` is invisible in review and only one of them is right. + vi.mocked(prisma.apiToken.count).mockResolvedValue(9 as never); + const res = await POST(req({ name: "Import script" }) as never); + expect(res.status).toBe(201); + }); + + it("checks the ceiling only after the body validates", async () => { + // A malformed request should hear about the malformation, not the + // ceiling — and should not spend a query establishing it. + vi.mocked(prisma.apiToken.count).mockResolvedValue(10 as never); + const res = await POST(req({}) as never); + expect(res.status).toBe(422); + expect(prisma.apiToken.count).not.toHaveBeenCalled(); + }); + + it("ignores a scope the body tries to smuggle in", async () => { + // There is no `scope` field to validate, so an unknown key is simply not + // read — the permission array is a literal either way. Asserted because + // "the field does not exist" is the security property, and a schema change + // that added one would silently make this body meaningful. + await POST( + req({ + name: "Import script", + scope: "read_write", + permissions: ["*"], + }) as never, + ); + expect(vi.mocked(issueApiToken).mock.calls[0][0].permissions).toEqual([ + DOCUMENTS_WRITE_SCOPE, + ]); + }); +}); diff --git a/src/app/api/tokens/documents/route.ts b/src/app/api/tokens/documents/route.ts new file mode 100644 index 000000000..0a5121047 --- /dev/null +++ b/src/app/api/tokens/documents/route.ts @@ -0,0 +1,127 @@ +/** + * `POST /api/tokens/documents` — mint a Bearer that can upload documents into + * the vault and do nothing else (#1038). + * + * The door another document system pushes through: a Paperless-ngx workflow + * posting a newly tagged document, or the import script copying an archive + * across. It is the measurement-ingest mint (`../measurements/route.ts`) + * copied for a second scope, and every argument made there holds here — read + * that file for the reasoning behind each guard. In short: + * + * - `permissions` is a literal, so no request shape reaches it; + * - `documents:write` is accepted by exactly one route, the upload, on the + * holder's own record, and the answer it gets back is a receipt rather than + * the stored row — it cannot list, read, download, edit or delete a + * document, reach the AI legs, or mint another token; + * - minting takes a cookie session, because a short-lived native access token + * must not be able to leave behind a credential that lives a year; + * - ten mints a minute and ten live tokens bound a runaway loop. + * + * No listing and no revoke of its own: `GET /api/tokens` lists every token + * with its permissions and `DELETE /api/tokens/[id]` revokes one. + */ +import { NextRequest } from "next/server"; + +import { apiHandler, requireCookieAuth } from "@/lib/api-handler"; +import { + apiError, + apiSuccess, + getClientIp, + returnAllZodIssues, + safeJson, +} from "@/lib/api-response"; +import { annotate } from "@/lib/logging/context"; +import { auditLog } from "@/lib/auth/audit"; +import { prisma } from "@/lib/db"; +import { issueApiToken } from "@/lib/auth/issue-token"; +import { isApiGloballyEnabled } from "@/lib/app-settings"; +import { checkRateLimit } from "@/lib/rate-limit"; +import { DOCUMENTS_WRITE_SCOPE } from "@/lib/documents/scopes"; +import { createDocumentTokenSchema } from "@/lib/validations/tokens"; + +/** + * Days a token lives when the caller names no lifetime. A year, like the + * measurement token: it is pasted into a workflow that runs unattended, where + * a quiet expiry surfaces as "new letters stopped arriving in the spring". + */ +const DEFAULT_EXPIRY_DAYS = 365; + +/** Mints per user per minute. */ +const MINT_RATE_LIMIT_MAX = 10; +const MINT_RATE_LIMIT_WINDOW_MS = 60 * 1000; + +/** Live document tokens an account may hold at once; catches a loop. */ +const MAX_LIVE_TOKENS = 10; + +export const POST = apiHandler(async (request: NextRequest) => { + const { user } = await requireCookieAuth(); + annotate({ action: { name: "tokens.documents.create" } }); + + if (!(await isApiGloballyEnabled())) { + return apiError("API is globally disabled", 403); + } + + const rl = await checkRateLimit( + `tokens:documents:mint:${user.id}`, + MINT_RATE_LIMIT_MAX, + MINT_RATE_LIMIT_WINDOW_MS, + ); + if (!rl.allowed) { + return apiError("Too many token mints, try again later", 429); + } + + const { data: body, error: jsonError } = await safeJson(request, { + maxBytes: 16 * 1024, + }); + if (jsonError) return jsonError; + + const parsed = createDocumentTokenSchema.safeParse(body); + if (!parsed.success) { + return returnAllZodIssues(parsed.error, 422); + } + + // Counted live and per scope, after validation and before the create; the + // count-then-create race is bounded by the mint bucket above. See the + // measurement mint for the full argument. + const live = await prisma.apiToken.count({ + where: { + userId: user.id, + revoked: false, + permissions: { has: DOCUMENTS_WRITE_SCOPE }, + OR: [{ expiresAt: null }, { expiresAt: { gt: new Date() } }], + }, + }); + if (live >= MAX_LIVE_TOKENS) { + annotate({ + action: { name: "tokens.documents.create" }, + meta: { outcome: "ceiling_reached", live_token_count: live }, + }); + return apiError( + `You already have ${MAX_LIVE_TOKENS} document tokens. Revoke one before creating another.`, + 409, + { errorCode: "tokens.documents.ceiling_reached" }, + ); + } + + const issued = await issueApiToken({ + userId: user.id, + name: parsed.data.name, + // A literal, never spread and never derived from the body. `issueApiToken` + // defaults to `["*"]` when this property is absent, which is why the unit + // suite asserts this array and not merely the 201. + permissions: [DOCUMENTS_WRITE_SCOPE], + expiresInDays: parsed.data.expiresInDays ?? DEFAULT_EXPIRY_DAYS, + }); + + await auditLog("tokens.documents.create", { + userId: user.id, + ipAddress: getClientIp(request), + details: { tokenId: issued.tokenId, scope: DOCUMENTS_WRITE_SCOPE }, + }); + + // The raw token, once. It is stored as an HMAC and no path re-reveals it. + return apiSuccess( + { token: issued.token, name: issued.name, expiresAt: issued.expiresAt }, + 201, + ); +}); diff --git a/src/components/documents/__tests__/document-card.test.tsx b/src/components/documents/__tests__/document-card.test.tsx index 7b0aa82a1..8de53bd90 100644 --- a/src/components/documents/__tests__/document-card.test.tsx +++ b/src/components/documents/__tests__/document-card.test.tsx @@ -37,6 +37,8 @@ function doc(overrides: Partial = {}): InboundDocumentDto { lastIndexAttemptAt: null, lastIndexOutcome: null, hasThumbnail: false, + sourceSystem: null, + sourceId: null, createdAt: "2025-10-05T08:00:00.000Z", updatedAt: "2025-10-05T08:00:00.000Z", ...overrides, diff --git a/src/components/documents/__tests__/document-facts-review.test.tsx b/src/components/documents/__tests__/document-facts-review.test.tsx index 7ab4798ef..c2c64d314 100644 --- a/src/components/documents/__tests__/document-facts-review.test.tsx +++ b/src/components/documents/__tests__/document-facts-review.test.tsx @@ -89,6 +89,8 @@ function doc( lastIndexAttemptAt: null, lastIndexOutcome: null, hasThumbnail: false, + sourceSystem: null, + sourceId: null, createdAt: "2026-08-01T10:00:00.000Z", updatedAt: "2026-08-01T10:00:00.000Z", facts: [fact()], diff --git a/src/components/documents/__tests__/episode-documents-card.test.tsx b/src/components/documents/__tests__/episode-documents-card.test.tsx index db5856609..6df1c1dd7 100644 --- a/src/components/documents/__tests__/episode-documents-card.test.tsx +++ b/src/components/documents/__tests__/episode-documents-card.test.tsx @@ -57,6 +57,8 @@ function doc(id: string, title: string): InboundDocumentDto { lastIndexAttemptAt: null, lastIndexOutcome: null, hasThumbnail: false, + sourceSystem: null, + sourceId: null, createdAt: "2025-10-05T08:00:00.000Z", updatedAt: "2025-10-05T08:00:00.000Z", }; diff --git a/src/components/documents/__tests__/vault-utils.test.ts b/src/components/documents/__tests__/vault-utils.test.ts index 201963f85..ace2e807e 100644 --- a/src/components/documents/__tests__/vault-utils.test.ts +++ b/src/components/documents/__tests__/vault-utils.test.ts @@ -45,6 +45,8 @@ function doc(overrides: Partial = {}): InboundDocumentDto { lastIndexAttemptAt: null, lastIndexOutcome: null, hasThumbnail: false, + sourceSystem: null, + sourceId: null, createdAt: "2026-03-11T08:00:00.000Z", updatedAt: "2026-03-11T08:00:00.000Z", ...overrides, diff --git a/src/components/documents/document-detail-sheet.tsx b/src/components/documents/document-detail-sheet.tsx index 780105bfa..070fcec12 100644 --- a/src/components/documents/document-detail-sheet.tsx +++ b/src/components/documents/document-detail-sheet.tsx @@ -73,6 +73,7 @@ import { import { cn } from "@/lib/utils"; import { INBOUND_DOCUMENT_KINDS, + type DocumentSourceSystemValue, type DocumentSuggestionDto, type DocumentSummaryMode, type InboundDocumentDetailDto, @@ -698,6 +699,14 @@ export function DocumentDetailSheet({ : null; const title = doc?.title ?? doc?.filename ?? t("documents.card.untitled"); + // The product names are proper nouns and stay untranslated; only "another + // system" is prose. + const sourceSystemName = (system: DocumentSourceSystemValue): string => + system === "PAPERLESS" + ? "Paperless-ngx" + : system === "PAPRA" + ? "Papra" + : t("documents.detail.sourceOther"); const Icon = doc ? DOCUMENT_KIND_ICONS[doc.kind] : null; // v1.37.0 — the ONE eager fetch of the original, and only for a document // that is going to render it. @@ -1167,6 +1176,25 @@ export function DocumentDetailSheet({ })} {doc.filename ? ` · ${doc.filename}` : ""}

+ + {/* Where an imported document came from (#1038). Meta, so + muted like the line above; the id is what the person looks + up in the other system. */} + {doc.sourceSystem ? ( +

+ {doc.sourceId + ? t("documents.detail.importedFromWithId", { + system: sourceSystemName(doc.sourceSystem), + id: doc.sourceId, + }) + : t("documents.detail.importedFrom", { + system: sourceSystemName(doc.sourceSystem), + })} +

+ ) : null} ) : null} diff --git a/src/components/settings/api-section.tsx b/src/components/settings/api-section.tsx index 29ba7c2b7..c129059b6 100644 --- a/src/components/settings/api-section.tsx +++ b/src/components/settings/api-section.tsx @@ -6,10 +6,12 @@ import { Check, ChevronDown, Copy, + FileUp, Key, Loader2, Trash2, Upload, + type LucideIcon, } from "lucide-react"; import { toast } from "sonner"; @@ -55,6 +57,7 @@ export function ApiSection() {
+
); @@ -70,6 +73,64 @@ export function ApiSection() { * below is where credentials are reviewed and killed. */ function MeasurementTokensCard() { + const { t } = useTranslations(); + return ( + + ); +} + +/** + * Mint a token another document system uploads through (#1038): a Paperless + * workflow or the import script. Same card, same no-list reasoning as the + * measurement one above. + */ +function DocumentTokensCard() { + const { t } = useTranslations(); + return ( + + ); +} + +/** One single-scope mint: a name field, the token shown once, a copy button. */ +function IngestTokenCard({ + endpoint, + icon, + slot, + copy, +}: { + endpoint: "/api/tokens/measurements" | "/api/tokens/documents"; + icon: LucideIcon; + slot: string; + copy: { + title: string; + description: string; + detail: string; + scopeNote: string; + createFailed: string; + }; +}) { const { t } = useTranslations(); const queryClient = useQueryClient(); const [newName, setNewName] = useState(""); @@ -84,7 +145,7 @@ function MeasurementTokensCard() { setTokenMsg(null); setNewToken(null); try { - const res = await apiFetchRaw("/api/tokens/measurements", { + const res = await apiFetchRaw(endpoint, { method: "POST", headers: { "Content-Type": "application/json" }, // No scope field: the endpoint mints one shape, so there is nothing @@ -101,7 +162,7 @@ function MeasurementTokensCard() { // The server's own message when it sent one — it names the actual // cause (switch off, rate limit, validation) — and a message about // this specific action when it did not. - setTokenMsg(json.error || t("settings.measurementsToken.createFailed")); + setTokenMsg(json.error || copy.createFailed); } } catch { setTokenMsg(t("common.networkError")); @@ -125,17 +186,15 @@ function MeasurementTokensCard() { return (
-

- {t("settings.measurementsToken.detail")} -

+

{copy.detail}

- {t("settings.measurementsToken.scopeNote")} + {copy.scopeNote}

@@ -162,7 +221,7 @@ function MeasurementTokensCard() { {newToken && (

{t("settings.tokenCreated")} @@ -223,6 +282,12 @@ function ApiEndpointsCard() { auth: "Authorization: Bearer hlk_...", example: `{ "entries": [ { "hkIdentifier": "...", "value": 1, ... } ] }`, }, + { + method: "POST", + path: "/api/documents/inbound", + auth: "Authorization: Bearer hlk_...", + example: `multipart: file, title, documentDate, sourceSystem, sourceId`, + }, ]; return ( diff --git a/src/lib/data-wipe/wipe-plan.ts b/src/lib/data-wipe/wipe-plan.ts index e4d1b863b..a0e3ab52a 100644 --- a/src/lib/data-wipe/wipe-plan.ts +++ b/src/lib/data-wipe/wipe-plan.ts @@ -147,6 +147,11 @@ export const WIPE_MODELS = [ "DocumentContentIndex", "DocumentThumbnail", "DocumentConditionLink", + // The memory of which imported documents the person deleted (#1038). The + // documents themselves go below; keeping the memory would make a fresh + // import after the wipe skip files for a decision about a record that no + // longer exists. + "DocumentImportKey", "ExtractedFact", "ImportJob", diff --git a/src/lib/documents/__tests__/index-document.test.ts b/src/lib/documents/__tests__/index-document.test.ts index c9fab4ae9..40fb7409d 100644 --- a/src/lib/documents/__tests__/index-document.test.ts +++ b/src/lib/documents/__tests__/index-document.test.ts @@ -169,6 +169,23 @@ describe("indexDocumentContent — provider-first path", () => { }); describe("indexDocumentContent — local fallback path", () => { + it("a deferred import stays local even with an external pick and the toggle ON", async () => { + // #1038 — `aiRead=defer`: the person asked for AI reading to wait. + vi.mocked(resolveDocumentVisionProvider).mockResolvedValue(PICK as never); + visionOk(); + const outcome = await indexDocumentContent("user-1", "doc-1", { + localOnly: true, + }); + expect(outcome).toEqual({ + indexed: true, + source: "local-pdf", + tokenCount: 5, + }); + expect(resolveDocumentVisionProvider).not.toHaveBeenCalled(); + expect(transcribeDocument).not.toHaveBeenCalled(); + expect(reserveBudget).not.toHaveBeenCalled(); + }); + it("falls back to local when NO provider is configured", async () => { vi.mocked(resolveDocumentVisionProvider).mockResolvedValue({ chain: [], diff --git a/src/lib/documents/__tests__/upload-token-limit.test.ts b/src/lib/documents/__tests__/upload-token-limit.test.ts new file mode 100644 index 000000000..b263b5365 --- /dev/null +++ b/src/lib/documents/__tests__/upload-token-limit.test.ts @@ -0,0 +1,40 @@ +/** + * The per-token upload ceiling for `documents:write` callers (#1038) is an + * operator setting (`DOCUMENT_UPLOAD_LIMIT_PER_HOUR`), not a constant. Pins the + * default, the clamp, and the fallback on garbage — the same contract as + * `DOCUMENT_AI_LIMIT_PER_HOUR`. + */ +import { afterEach, describe, expect, it } from "vitest"; + +import { resolveDocumentUploadLimitPerHour } from "../upload-policy"; + +const ORIGINAL = process.env.DOCUMENT_UPLOAD_LIMIT_PER_HOUR; + +afterEach(() => { + if (ORIGINAL === undefined) delete process.env.DOCUMENT_UPLOAD_LIMIT_PER_HOUR; + else process.env.DOCUMENT_UPLOAD_LIMIT_PER_HOUR = ORIGINAL; +}); + +describe("resolveDocumentUploadLimitPerHour", () => { + it("defaults to 120 when unset", () => { + delete process.env.DOCUMENT_UPLOAD_LIMIT_PER_HOUR; + expect(resolveDocumentUploadLimitPerHour()).toBe(120); + }); + + it("honours an operator value", () => { + process.env.DOCUMENT_UPLOAD_LIMIT_PER_HOUR = "400"; + expect(resolveDocumentUploadLimitPerHour()).toBe(400); + }); + + it("falls back to the default on a non-number", () => { + process.env.DOCUMENT_UPLOAD_LIMIT_PER_HOUR = "lots"; + expect(resolveDocumentUploadLimitPerHour()).toBe(120); + }); + + it("clamps to 1-1000", () => { + process.env.DOCUMENT_UPLOAD_LIMIT_PER_HOUR = "0"; + expect(resolveDocumentUploadLimitPerHour()).toBe(1); + process.env.DOCUMENT_UPLOAD_LIMIT_PER_HOUR = "50000"; + expect(resolveDocumentUploadLimitPerHour()).toBe(1000); + }); +}); diff --git a/src/lib/documents/index-document.ts b/src/lib/documents/index-document.ts index a35e24ca7..faf8e984e 100644 --- a/src/lib/documents/index-document.ts +++ b/src/lib/documents/index-document.ts @@ -383,9 +383,22 @@ export async function indexLoadedDocument( export async function indexDocumentContent( userId: string, documentId: string, + options: { localOnly?: boolean } = {}, ): Promise { const document = await loadOwnedDocument(userId, documentId); if (!document) return { indexed: false, reason: "not-found" }; - const provider = await resolveIndexProvider(userId); + // v1.39.2 (#1038) — an import that asked for `aiRead=defer` gets the + // provider-free text layer only. An empty context is the same shape "no + // provider configured" already produces, so the tree falls straight to the + // local path without resolving a chain it would not use. + const provider: ResolvedIndexProvider = options.localOnly + ? { + chain: [], + pick: null, + consentOk: false, + dailyCap: 0, + costOwner: "operator", + } + : await resolveIndexProvider(userId); return indexLoadedDocument(userId, document, provider); } diff --git a/src/lib/documents/scopes.ts b/src/lib/documents/scopes.ts new file mode 100644 index 000000000..036d46fa2 --- /dev/null +++ b/src/lib/documents/scopes.ts @@ -0,0 +1,16 @@ +/** + * The Bearer scope for pushing documents into the vault from another system. + * + * A zero-import leaf for the reason `MEASUREMENTS_WRITE_SCOPE` is one (see + * `@/lib/measurements/scopes`): the mint route, the upload route, the settings + * card and the guards all read it, and none of them should inherit an import + * graph for a string. + * + * Deliberately alone. There is no read counterpart and must not be one: the + * list, detail, original, thumbnail, bulk and AI legs of the vault stay + * cookie-equivalent, so a credential pasted into a Paperless workflow or an + * import script can add a file and learn nothing about the vault it lands in. + */ + +/** The scope a narrow token must carry to upload a document. */ +export const DOCUMENTS_WRITE_SCOPE = "documents:write"; diff --git a/src/lib/documents/store.ts b/src/lib/documents/store.ts index 57c9cbbde..268504ed7 100644 --- a/src/lib/documents/store.ts +++ b/src/lib/documents/store.ts @@ -17,7 +17,10 @@ import type { InboundDocumentStatus, } from "@/generated/prisma/client"; import { servingClassFor } from "@/lib/documents/upload-policy"; -import { toIndexOutcome } from "@/lib/validations/inbound-documents"; +import { + toDocumentSourceSystem, + toIndexOutcome, +} from "@/lib/validations/inbound-documents"; import type { DocumentConditionLinkDto, DocumentEncounterLinkDto, @@ -247,6 +250,8 @@ export function serialiseDocument( ? null : toIndexOutcome(doc.lastIndexOutcome), hasThumbnail, + sourceSystem: toDocumentSourceSystem(doc.sourceSystem), + sourceId: doc.sourceSystem ? doc.sourceId : null, createdAt: doc.createdAt.toISOString(), updatedAt: doc.updatedAt.toISOString(), }; diff --git a/src/lib/documents/upload-policy.ts b/src/lib/documents/upload-policy.ts index 5a7ad1e92..55e114f5a 100644 --- a/src/lib/documents/upload-policy.ts +++ b/src/lib/documents/upload-policy.ts @@ -84,6 +84,32 @@ export function acquireDocumentUploadSlot(userId: string): (() => void) | null { }; } +/** + * Hourly upload ceiling for a narrow `documents:write` token, per token. + * + * Its own bucket rather than the person's: an overnight import running at this + * rate must never use up the allowance the web app and the phone draw on. The + * default is twice the cookie bucket because an importer is strictly serial + * and is the reason the token exists. Operator-tunable via + * `DOCUMENT_UPLOAD_LIMIT_PER_HOUR`, clamped to 1-1000 the way + * `DOCUMENT_AI_LIMIT_PER_HOUR` is, so a typo can neither shut the door nor + * remove the cap. The per-user quota, not this number, is the disk guard. + */ +const DOCUMENT_UPLOAD_TOKEN_LIMIT_DEFAULT = 120; +const DOCUMENT_UPLOAD_TOKEN_LIMIT_MIN = 1; +const DOCUMENT_UPLOAD_TOKEN_LIMIT_MAX = 1000; + +export function resolveDocumentUploadLimitPerHour(): number { + const raw = process.env.DOCUMENT_UPLOAD_LIMIT_PER_HOUR; + if (!raw) return DOCUMENT_UPLOAD_TOKEN_LIMIT_DEFAULT; + const parsed = Number.parseInt(raw, 10); + if (!Number.isFinite(parsed)) return DOCUMENT_UPLOAD_TOKEN_LIMIT_DEFAULT; + return Math.min( + DOCUMENT_UPLOAD_TOKEN_LIMIT_MAX, + Math.max(DOCUMENT_UPLOAD_TOKEN_LIMIT_MIN, parsed), + ); +} + export interface DocumentLimits { /** Per-file upload cap in bytes (admin-tunable, clamped to the ceiling). */ maxFileBytes: number; diff --git a/src/lib/export/backup-plan.ts b/src/lib/export/backup-plan.ts index b1e9a3a17..e84d1bb48 100644 --- a/src/lib/export/backup-plan.ts +++ b/src/lib/export/backup-plan.ts @@ -670,6 +670,8 @@ export const NOT_IN_BACKUP_MODELS: Readonly> = { "The record of a restore this host ran from one of its own stored copies. It names a backup row that exists only in this database, so carrying it would describe a restore the receiving host never performed.", OffhostBackupState: "When this host last put this account's copy in this operator's bucket, and how big it was. It describes one deployment's relationship with one bucket, so restoring it elsewhere would assert an off-host copy that host has never written.", + DocumentImportKey: + "The source keys of imported documents the person deleted and the purge then removed, kept so a re-run of the importer does not store them again. The backup carries only live documents, so the tombstones that hold the same memory before the purge do not travel either; carrying the ledger alone would restore half of one decision. After a restore, a deliberate re-import can bring such a document back, and the person deletes it again.", ImportJob: "A job record pointing at an uploaded file that the backup does not carry, so restoring it would resurrect a task with nothing to work on.", InviteToken: diff --git a/src/lib/export/records-backup.ts b/src/lib/export/records-backup.ts index 2948689e8..e1d6026c6 100644 --- a/src/lib/export/records-backup.ts +++ b/src/lib/export/records-backup.ts @@ -171,6 +171,12 @@ export interface DocumentBackupEntry { /** Refs #776 — the index-attempt record, carried verbatim (canonical DR). */ lastIndexAttemptAt?: string | null; lastIndexOutcome?: string | null; + /** + * v1.39.2 (#1038) — where an imported document came from, both payload + * kinds. Optional so a file written before the fields existed still parses. + */ + sourceSystem?: string | null; + sourceId?: string | null; } export interface RecordsBackupOptions { @@ -296,6 +302,8 @@ export async function buildRecordsBackupSection( summaryState: true, lastIndexAttemptAt: true, lastIndexOutcome: true, + sourceSystem: true, + sourceId: true, createdAt: true, updatedAt: true, }, @@ -314,6 +322,8 @@ export async function buildRecordsBackupSection( reportDate: true, documentDate: true, summaryEncrypted: true, + sourceSystem: true, + sourceId: true, createdAt: true, }, }); @@ -520,6 +530,8 @@ export async function buildRecordsBackupSection( documentDate: d.documentDate ? d.documentDate.toISOString().slice(0, 10) : null, + sourceSystem: d.sourceSystem, + sourceId: d.sourceId, createdAt: d.createdAt.toISOString(), }; diff --git a/src/lib/export/restore-backup.ts b/src/lib/export/restore-backup.ts index 596d60ab9..123e6c255 100644 --- a/src/lib/export/restore-backup.ts +++ b/src/lib/export/restore-backup.ts @@ -2019,6 +2019,10 @@ export async function restoreBackup( ? new Date(document.lastIndexAttemptAt) : null, lastIndexOutcome: document.lastIndexOutcome ?? null, + sourceSystem: document.sourceSystem ?? null, + sourceId: document.sourceSystem + ? (document.sourceId ?? null) + : null, createdAt: new Date(document.createdAt!), updatedAt: new Date(document.updatedAt!), })), diff --git a/src/lib/jobs/__tests__/document-index.test.ts b/src/lib/jobs/__tests__/document-index.test.ts index e175bc1a8..3d7652ae9 100644 --- a/src/lib/jobs/__tests__/document-index.test.ts +++ b/src/lib/jobs/__tests__/document-index.test.ts @@ -11,6 +11,9 @@ vi.mock("@/lib/documents/index-document", () => ({ })); vi.mock("@/lib/jobs/boss-instance", () => ({ getGlobalBoss: vi.fn() })); vi.mock("@/lib/logging/context", () => ({ annotate: vi.fn() })); +vi.mock("@/lib/documents/auto-stage-labs", () => ({ + maybeAutoStageLabFacts: vi.fn().mockResolvedValue(undefined), +})); import { DOCUMENT_INDEX_QUEUE, @@ -19,6 +22,7 @@ import { } from "../document-index"; import { indexDocumentContent } from "@/lib/documents/index-document"; import { getGlobalBoss } from "@/lib/jobs/boss-instance"; +import { maybeAutoStageLabFacts } from "@/lib/documents/auto-stage-labs"; beforeEach(() => { vi.clearAllMocks(); @@ -38,6 +42,15 @@ describe("enqueueDocumentIndex", () => { ); }); + it("carries localOnly for a deferred import, and only then", async () => { + const send = vi.fn().mockResolvedValue("job-1"); + vi.mocked(getGlobalBoss).mockReturnValue({ send } as never); + await enqueueDocumentIndex("user-1", "doc-1", { localOnly: true }); + await enqueueDocumentIndex("user-1", "doc-2"); + expect(send.mock.calls[0][1]).toMatchObject({ localOnly: true }); + expect(send.mock.calls[1][1]).not.toHaveProperty("localOnly"); + }); + it("no-ops when the boss is not running", async () => { vi.mocked(getGlobalBoss).mockReturnValue(null as never); const result = await enqueueDocumentIndex("user-1", "doc-1"); @@ -62,7 +75,27 @@ describe("runDocumentIndex", () => { tokenCount: 3, } as never); await runDocumentIndex({ userId: "user-1", documentId: "doc-1" }); - expect(indexDocumentContent).toHaveBeenCalledWith("user-1", "doc-1"); + expect(indexDocumentContent).toHaveBeenCalledWith("user-1", "doc-1", { + localOnly: false, + }); + expect(maybeAutoStageLabFacts).toHaveBeenCalledWith("user-1", "doc-1"); + }); + + it("a deferred import indexes locally and stages no lab facts", async () => { + vi.mocked(indexDocumentContent).mockResolvedValue({ + indexed: true, + source: "local-pdf", + tokenCount: 3, + } as never); + await runDocumentIndex({ + userId: "user-1", + documentId: "doc-1", + localOnly: true, + }); + expect(indexDocumentContent).toHaveBeenCalledWith("user-1", "doc-1", { + localOnly: true, + }); + expect(maybeAutoStageLabFacts).not.toHaveBeenCalled(); }); it("ignores a payload missing ids without calling the tree", async () => { diff --git a/src/lib/jobs/document-index.ts b/src/lib/jobs/document-index.ts index d6cbf4b4a..74f16d12d 100644 --- a/src/lib/jobs/document-index.ts +++ b/src/lib/jobs/document-index.ts @@ -26,6 +26,12 @@ export interface DocumentIndexPayload { userId: string; documentId: string; enqueuedAt?: string; + /** + * v1.39.2 (#1038) — set for an upload that asked for `aiRead=defer`: index + * from the local text layer only and stage no lab facts. Absent on every + * job enqueued before the field existed, which keeps their behaviour. + */ + localOnly?: boolean; } /** @@ -38,7 +44,10 @@ export async function runDocumentIndex( ): Promise { const { userId, documentId } = payload; if (!userId || !documentId) return; - const outcome = await indexDocumentContent(userId, documentId); + const localOnly = payload.localOnly === true; + const outcome = await indexDocumentContent(userId, documentId, { + localOnly, + }); annotate({ action: { name: "documents.autoIndex.run" }, meta: outcome.indexed @@ -51,7 +60,9 @@ export async function runDocumentIndex( // PENDING for the existing human review; nothing is committed. Fully gated + // idempotent inside; a failure here must never fail the index job, so it is // swallowed to a no-op (the manual extract button remains the fallback). - if (outcome.indexed) { + // A deferred import stages nothing: lab staging is an AI read, and the + // person asked for those to wait. + if (outcome.indexed && !localOnly) { await maybeAutoStageLabFacts(userId, documentId).catch(() => {}); } } @@ -67,6 +78,7 @@ export async function runDocumentIndex( export async function enqueueDocumentIndex( userId: string, documentId: string, + options: { localOnly?: boolean } = {}, ): Promise<{ enqueued: boolean }> { const boss = getGlobalBoss(); if (!boss) return { enqueued: false }; @@ -74,6 +86,7 @@ export async function enqueueDocumentIndex( userId, documentId, enqueuedAt: new Date().toISOString(), + ...(options.localOnly ? { localOnly: true } : {}), }; try { const jobId = await boss.send(DOCUMENT_INDEX_QUEUE, payload, { diff --git a/src/lib/jobs/document-purge.ts b/src/lib/jobs/document-purge.ts index eebb5eb7e..c63e211c0 100644 --- a/src/lib/jobs/document-purge.ts +++ b/src/lib/jobs/document-purge.ts @@ -7,7 +7,8 @@ * grace window — returning the TOAST space the encrypted blob holds (a * tombstoned document keeps counting against the quota until purged, so * "deleted" bytes are never invisible backup weight). Staged facts cascade - * via the FK. Restore-after-purge answers 409 at the API. + * via the FK. Restore-after-purge answers 409 at the API. An imported + * document's source key is carried to `DocumentImportKey` on the way out. */ import { type Job } from "pg-boss"; import { withBackgroundEvent } from "@/lib/logging/background"; @@ -38,10 +39,34 @@ export async function purgeExpiredDocumentTombstones( now: Date = new Date(), ): Promise { const cutoff = new Date(now.getTime() - DOCUMENT_PURGE_GRACE_DAYS * DAY_MS); - const { count } = await prisma.inboundDocument.deleteMany({ - where: { deletedAt: { not: null, lt: cutoff } }, + const expired = { deletedAt: { not: null, lt: cutoff } }; + return prisma.$transaction(async (tx) => { + // v1.39.2 (#1038) — an imported document's source key outlives its row. + // While the tombstone exists the key sits on it and the upload route + // answers "deleted"; once the row goes, this ledger is what keeps a re-run + // of the importer from storing the document again. Same transaction as the + // delete, so a purge that fails halfway forgets nothing. + const keyed = await tx.inboundDocument.findMany({ + where: { + ...expired, + sourceSystem: { not: null }, + sourceId: { not: null }, + }, + select: { userId: true, sourceSystem: true, sourceId: true }, + }); + if (keyed.length > 0) { + await tx.documentImportKey.createMany({ + data: keyed.map((row) => ({ + userId: row.userId, + sourceSystem: row.sourceSystem!, + sourceId: row.sourceId!, + })), + skipDuplicates: true, + }); + } + const { count } = await tx.inboundDocument.deleteMany({ where: expired }); + return count; }); - return count; } export async function handleDocumentPurge( diff --git a/src/lib/openapi/error-codes.ts b/src/lib/openapi/error-codes.ts index b3a1e4d1b..64548daa8 100644 --- a/src/lib/openapi/error-codes.ts +++ b/src/lib/openapi/error-codes.ts @@ -312,7 +312,10 @@ export const ERROR_CODE_CATALOGUE: Readonly> = "source-priority.body.invalid_json", "source-priority.body.invalid_shape", ], - tokens: ["tokens.measurements.ceiling_reached"], + tokens: [ + "tokens.documents.ceiling_reached", + "tokens.measurements.ceiling_reached", + ], vaccination: [ "vaccination.booster-invalid", "vaccination.booster-no-antigen", diff --git a/src/lib/openapi/routes/auth.ts b/src/lib/openapi/routes/auth.ts index a5c80ad19..1df96d83f 100644 --- a/src/lib/openapi/routes/auth.ts +++ b/src/lib/openapi/routes/auth.ts @@ -30,7 +30,10 @@ import { mfaWebauthnLoginVerifySchema, } from "@/lib/validations/mfa"; import { oidcNativeTokenSchema } from "@/lib/validations/oidc-native"; -import { createMeasurementTokenSchema } from "@/lib/validations/tokens"; +import { + createDocumentTokenSchema, + createMeasurementTokenSchema, +} from "@/lib/validations/tokens"; import { nativeHandoffTokenSchema } from "@/lib/validations/native-handoff"; import { stepUpMintSchema, @@ -436,6 +439,28 @@ const createMeasurementTokenResponse = z "The one response that carries a usable measurement-ingest Bearer value.", }); +const createDocumentTokenRequest = createDocumentTokenSchema.meta({ + id: "CreateDocumentTokenRequest", + description: + "Mint a document-upload Bearer. No `scope` field: the endpoint mints one shape. `name` is what the token is listed under; name it after the system it is pasted into. `expiresInDays` defaults to 365.", +}); + +const createDocumentTokenResponse = z + .object({ + token: z + .string() + .describe( + "The raw `hlk_` Bearer, RETURNED EXACTLY ONCE. It is stored only as an HMAC-SHA256 hash and cannot be retrieved later. Treat it as a secret.", + ), + name: z.string(), + expiresAt: z.iso.datetime({ offset: true }), + }) + .meta({ + id: "CreateDocumentTokenResponse", + description: + "The one response that carries a usable document-upload Bearer value.", + }); + // ── Registration, credential rotation, Codex device-auth ───────────── const registerRequest = registerSchema.meta({ @@ -1536,7 +1561,7 @@ export const authPaths: NonNullable = { summary: "List the caller's API tokens", description: "Every `ApiToken` row belonging to the caller, newest first, revoked ones included — presence in this list is not validity, `revoked` is. The hash is never returned and there is no path that re-reveals a token's plaintext.\n\n" + - "Three things the name understates. The list is not limited to tokens a person minted from the settings surface: a native login mints a wildcard access token as an `ApiToken` row, so a signed-in phone shows up here too. There is no GENERIC mint any more — the POST at this exact path issued `[\"medication:ingest\"]` and was removed, because that scope reached no ingest route while the pre-fail-closed default let it reach everything else. The credentials that work are minted where they are scoped: the per-medication API-endpoint toggle, `POST /api/mcp/tokens` for a connector, and the sibling `POST /api/tokens/measurements` for third-party measurement ingest. And unlike the revoke, this read is NOT gated on the operator's instance-wide API switch: that switch governs the surfaces a token is for, not a token's ability to authenticate, so tokens stay live while it is off and their owner has to be able to see them.", + "Three things the name understates. The list is not limited to tokens a person minted from the settings surface: a native login mints a wildcard access token as an `ApiToken` row, so a signed-in phone shows up here too. There is no GENERIC mint any more — the POST at this exact path issued `[\"medication:ingest\"]` and was removed, because that scope reached no ingest route while the pre-fail-closed default let it reach everything else. The credentials that work are minted where they are scoped: the per-medication API-endpoint toggle, `POST /api/mcp/tokens` for a connector, the sibling `POST /api/tokens/measurements` for third-party measurement ingest, and `POST /api/tokens/documents` for document upload from another system. And unlike the revoke, this read is NOT gated on the operator's instance-wide API switch: that switch governs the surfaces a token is for, not a token's ability to authenticate, so tokens stay live while it is off and their owner has to be able to see them.", responses: { "200": { description: "The caller's tokens, newest first.", @@ -1620,6 +1645,70 @@ export const authPaths: NonNullable = { }, }, }, + "/api/tokens/documents": { + post: { + tags: ["Auth"], + summary: "Mint a document-upload token", + description: + "Mints a Bearer scoped to exactly `documents:write` and audits the mint. THE RESPONSE CARRIES THE RAW TOKEN, once; it is stored as an HMAC and no path re-reveals it.\n\n" + + "**What it can do.** `POST /api/documents/inbound` on its owner's own record, and nothing else. It is the door another document system pushes through: a Paperless-ngx workflow posting a newly tagged document, or an import script copying an archive. Uploads through it draw on a bucket of their own, keyed on the token (default 120 an hour, operator-tunable via `DOCUMENT_UPLOAD_LIMIT_PER_HOUR`), and get back a receipt (`id`, `duplicate`, and `deleted` when the source key belongs to a document the owner deleted) rather than the stored row.\n\n" + + "**What it cannot do.** List, read, download, preview, edit, delete or bulk-act on a document, reach the document-AI routes, act on a shared record, or mint another token. Every other route names no scope and refuses it 403.\n\n" + + "Minting requires a COOKIE SESSION, for the reason the measurement mint gives: a day-lived native access token must not be able to leave behind a year-lived credential. Gated by the operator's instance-wide API switch. Body capped at 16 KiB; 10 mints per user per minute, and at most 10 live document tokens held at once. Tokens appear in `GET /api/tokens` and are revoked at `DELETE /api/tokens/{id}`.", + requestBody: { + required: true, + content: { + "application/json": { schema: createDocumentTokenRequest }, + }, + }, + responses: { + "201": { + description: + "Token minted. The `token` field is the only copy of the secret.", + content: { + "application/json": { + schema: dataEnvelope( + createDocumentTokenResponse, + "CreateDocumentTokenEnvelope", + ), + }, + }, + }, + "400": { + description: "Body is not parseable JSON.", + content: { "application/json": { schema: errorEnvelope } }, + }, + "403": { + description: + "The operator has switched the API off instance-wide. Nothing was minted. A Bearer caller is refused 401 before the switch is read.", + content: { "application/json": { schema: errorEnvelope } }, + }, + "413": { + description: "Body exceeds 16 KiB.", + content: { "application/json": { schema: errorEnvelope } }, + }, + "415": { + description: "Content-Type is not `application/json`.", + content: { "application/json": { schema: errorEnvelope } }, + }, + ...stdResponses, + "422": { + description: + "Validation failed. The envelope carries every offending issue.", + content: { "application/json": { schema: errorEnvelope } }, + }, + "429": { + description: + "More than 10 mints from this account in a minute. Nothing was minted.", + content: { "application/json": { schema: errorEnvelope } }, + }, + "409": { + description: + "The account already holds 10 live document tokens (`meta.errorCode` = `tokens.documents.ceiling_reached`). Nothing was minted. Revoke one at `DELETE /api/tokens/{id}`; only live tokens carrying `documents:write` count.", + content: { "application/json": { schema: errorEnvelope } }, + }, + }, + }, + }, "/api/tokens/{id}": { delete: { tags: ["Auth"], diff --git a/src/lib/openapi/routes/documents.ts b/src/lib/openapi/routes/documents.ts index 3ee9bbb2a..3f99bf02f 100644 --- a/src/lib/openapi/routes/documents.ts +++ b/src/lib/openapi/routes/documents.ts @@ -23,6 +23,8 @@ import { documentUpdateSchema, inboundConfirmSchema, inboundFactEditSchema, + DOCUMENT_SOURCE_ID_MAX, + DOCUMENT_SOURCE_SYSTEMS, DOCUMENT_SUMMARY_STATES, INBOUND_DOCUMENT_KINDS, INBOUND_DOCUMENT_STATUSES, @@ -204,13 +206,15 @@ const inboundDocument = z ]) .nullable(), hasThumbnail: z.boolean(), + sourceSystem: z.enum(DOCUMENT_SOURCE_SYSTEMS).nullable(), + sourceId: z.string().nullable(), createdAt: z.string(), updatedAt: z.string(), }) .meta({ id: "InboundDocument", description: - "A stored document. The raw bytes are stored encrypted at rest and never returned; this is the metadata + staging summary. `status` is STORED for a freshly uploaded file (no extraction run). `title` is the user label (plaintext); `documentDate` is the user filing date; `reportDate` is the model-transcribed date (null until extraction runs). `servingClass` says how `/original` delivers the file — render inline (`inline`) or download-only (`attachment`); render/download by it, never by MIME guess. `hasContentIndex` is true when the document has a content-search index (auto-indexed on upload); `contentIndexSource` says how that index was produced — `vision` means an AI provider read the original, the other values are provider-free local extractions, and it is null when `hasContentIndex` is false. `lastIndexAttemptAt` is when the most recent index attempt finished (successful or not; null when none ran); `lastIndexOutcome` says why that attempt produced no index, and is null when the attempt succeeded or none ran. `hasThumbnail` is true when a preview thumbnail has been rendered in the background (fetch it from `/api/documents/inbound/{id}/thumbnail`); false means no preview yet or an unsupported type — show a placeholder. Treat an unknown `kind` value as OTHER when decoding.", + "A stored document. The raw bytes are stored encrypted at rest and never returned; this is the metadata + staging summary. `status` is STORED for a freshly uploaded file (no extraction run). `title` is the user label (plaintext); `documentDate` is the user filing date; `reportDate` is the model-transcribed date (null until extraction runs). `servingClass` says how `/original` delivers the file — render inline (`inline`) or download-only (`attachment`); render/download by it, never by MIME guess. `hasContentIndex` is true when the document has a content-search index (auto-indexed on upload); `contentIndexSource` says how that index was produced — `vision` means an AI provider read the original, the other values are provider-free local extractions, and it is null when `hasContentIndex` is false. `lastIndexAttemptAt` is when the most recent index attempt finished (successful or not; null when none ran); `lastIndexOutcome` says why that attempt produced no index, and is null when the attempt succeeded or none ran. `hasThumbnail` is true when a preview thumbnail has been rendered in the background (fetch it from `/api/documents/inbound/{id}/thumbnail`); false means no preview yet or an unsupported type — show a placeholder. `sourceSystem` / `sourceId` say where an imported document came from and its id there (both null for a document added in HealthLog); treat an unknown `sourceSystem` as OTHER when decoding. Treat an unknown `kind` value as OTHER when decoding.", }); const inboundDocumentDetail = inboundDocument @@ -262,11 +266,23 @@ const confirmResponse = z const kindValues = [...INBOUND_DOCUMENT_KINDS]; +const documentUploadReceipt = z + .object({ + id: z.string().nullable(), + duplicate: z.boolean(), + deleted: z.literal(true).optional(), + }) + .meta({ + id: "DocumentUploadReceipt", + description: + "What an upload answers instead of the stored row. A narrow `documents:write` token always gets this shape (`duplicate: false` on 201, `true` on 200): a credential that can only add files learns whether a file is stored, not its title, filename or links. Every caller gets it when the upload's source key belongs to a document the owner DELETED (`deleted: true`, 200, nothing stored); `id` is the tombstoned row, or null once the 30-day purge removed it.", + }); + // Shared by the POST 200 (duplicate) and 201 (created) responses — the // envelope id must be registered exactly once. -const inboundDocumentEnvelope = dataEnvelope( - inboundDocument, - "InboundDocumentEnvelope", +const documentUploadEnvelope = dataEnvelope( + z.union([inboundDocument, documentUploadReceipt]), + "DocumentUploadEnvelope", ); export const inboundDocumentPaths: NonNullable = { @@ -387,7 +403,9 @@ export const inboundDocumentPaths: NonNullable = { tags: ["Documents"], summary: "Store a document (no extraction)", description: - 'STORE-ONLY upload. Stores the raw document ENCRYPTED at rest with `status: STORED` and runs NO extraction — provider-free, no AI consent / budget / egress. A file is always filable, even with no document-scan provider configured. `multipart/form-data`: a `file` plus optional `title`, `kind`, `documentDate` (YYYY-MM-DD), and repeated `episodeIds` form fields (pre-link to the caller\'s illness/condition episodes). Accepted types (magic-byte sniffed, never the wire Content-Type): PDF/JPEG/PNG/WebP/GIF render inline; Office (docx/xlsx/pptx/doc/xls/ppt), text/CSV/Markdown/RTF, TIFF, HEIC/HEIF, XML/JSON are stored verbatim and served download-only. HEIC is stored as-is but attachment-only — prefer transcoding to JPEG client-side for inline preview parity. Error contract: `413` with `meta.reason = "fileTooLarge"` (+ `maxFileBytes`) or `"quotaExceeded"` (+ `quotaBytes`, `usedBytes`); `415` with `meta.reason = "unsupportedType"`. A same-user duplicate (same bytes) returns 200 + `meta.duplicate: true` with the existing row — not an error. `Idempotency-Key` honoured. Read `GET /api/documents/inbound/usage` for the effective limits before offering an upload. AI extraction is a separate opt-in action — see `POST /api/documents/inbound/{id}/extract`.', + 'STORE-ONLY upload. Stores the raw document ENCRYPTED at rest with `status: STORED` and runs NO extraction — provider-free, no AI consent / budget / egress. A file is always filable, even with no document-scan provider configured. `multipart/form-data`: a `file` plus optional `title`, `kind`, `documentDate` (YYYY-MM-DD), and repeated `episodeIds` form fields (pre-link to the caller\'s illness/condition episodes). Accepted types (magic-byte sniffed, never the wire Content-Type): PDF/JPEG/PNG/WebP/GIF render inline; Office (docx/xlsx/pptx/doc/xls/ppt), text/CSV/Markdown/RTF, TIFF, HEIC/HEIF, XML/JSON are stored verbatim and served download-only. HEIC is stored as-is but attachment-only — prefer transcoding to JPEG client-side for inline preview parity. Error contract: `413` with `meta.reason = "fileTooLarge"` (+ `maxFileBytes`) or `"quotaExceeded"` (+ `quotaBytes`, `usedBytes`); `415` with `meta.reason = "unsupportedType"`. A same-user duplicate (same bytes) returns 200 + `meta.duplicate: true` with the existing row — not an error. `Idempotency-Key` honoured. Read `GET /api/documents/inbound/usage` for the effective limits before offering an upload. AI extraction is a separate opt-in action — see `POST /api/documents/inbound/{id}/extract`.\n\n' + + "**Importing from another system (v1.39.2).** Optional `sourceSystem` (`PAPERLESS`, `PAPRA`, `OTHER`) and `sourceId` (its id there, printable, up to 128 characters; needs `sourceSystem`) key the upload. A re-send with a known key is a duplicate: 200 with the live row (`meta.duplicate: true`), or, when the owner DELETED that document, 200 with `data.deleted: true` and `meta.deleted: true` and nothing stored; this holds after the 30-day purge too. `aiRead=defer` holds back automatic AI reading for this upload: the thumbnail and a local text index still run, the summary and lab staging do not, so the document waits for the person to read it deliberately. Absent, behaviour is unchanged.\n\n" + + "**Narrow token.** Also reachable with a `documents:write` Bearer (minted at `POST /api/tokens/documents`), and it is the only route that scope reaches. Such a caller draws on a bucket of its own, keyed on the token (default 120 an hour, `DOCUMENT_UPLOAD_LIMIT_PER_HOUR`, clamped 1-1000; the 429 carries `Retry-After` and `X-RateLimit-*`), and gets a `DocumentUploadReceipt` instead of the stored row. A cookie or wildcard caller keeps the 60-an-hour per-user bucket and the full row. With the vault module off the answer is 403 `module.disabled` for every caller.", requestBody: { required: true, content: { @@ -417,6 +435,18 @@ export const inboundDocumentPaths: NonNullable = { description: "Optional visit pre-links (repeat the field per id). Every id must be a live visit of the caller; an unknown one refuses the upload rather than dropping the link the person asked for.", }), + sourceSystem: z.enum(DOCUMENT_SOURCE_SYSTEMS).optional().meta({ + description: + "Optional. The system an imported document comes from. Shown on the detail view.", + }), + sourceId: z.string().max(DOCUMENT_SOURCE_ID_MAX).optional().meta({ + description: + "Optional. The document's id in `sourceSystem`; requires it. Unique per account across live and deleted documents.", + }), + aiRead: z.enum(["defer"]).optional().meta({ + description: + "Optional. `defer` holds back automatic AI reading for this upload (local index and thumbnail only).", + }), }), }, }, @@ -425,18 +455,19 @@ export const inboundDocumentPaths: NonNullable = { ...idempotentWrite(), "200": { description: - "Duplicate — the same bytes are already stored: the existing row, with `meta.duplicate: true` at the envelope level.", + "Duplicate — the same bytes, or the same source key, are already stored: the existing row (a `DocumentUploadReceipt` for a `documents:write` token), with `meta.duplicate: true` at the envelope level. When the source key belongs to a document the owner deleted: a receipt with `deleted: true` and `meta.deleted: true`, nothing stored.", content: { "application/json": { - schema: inboundDocumentEnvelope, + schema: documentUploadEnvelope, }, }, }, "201": { - description: "The stored document.", + description: + "The stored document (a `DocumentUploadReceipt` for a `documents:write` token).", content: { "application/json": { - schema: inboundDocumentEnvelope, + schema: documentUploadEnvelope, }, }, }, diff --git a/src/lib/validations/backup.ts b/src/lib/validations/backup.ts index d7abc0d74..088f066d7 100644 --- a/src/lib/validations/backup.ts +++ b/src/lib/validations/backup.ts @@ -81,6 +81,10 @@ import { emergencyBloodTypeSchema, organDonorStatusSchema, } from "@/lib/validations/emergency-profile"; +import { + DOCUMENT_SOURCE_ID_MAX, + DOCUMENT_SOURCE_SYSTEMS, +} from "@/lib/validations/inbound-documents"; import { REMINDER_EVENT_SOURCES } from "@/lib/measurement-reminders/satisfy"; import { UNITS_PER_DOSE_MESSAGE, @@ -1621,6 +1625,10 @@ const documentBackupSchema = z // Free string on purpose: a future outcome value must not fail a restore. lastIndexAttemptAt: isoDateTime.nullable().optional(), lastIndexOutcome: z.string().nullable().optional(), + // v1.39.2 (#1038) — import provenance. Closed like the upload field, so a + // restore cannot write a system the detail sheet has no name for. + sourceSystem: z.enum(DOCUMENT_SOURCE_SYSTEMS).nullable().optional(), + sourceId: z.string().max(DOCUMENT_SOURCE_ID_MAX).nullable().optional(), summary: z.string().nullable().optional(), createdAt: isoDateTime.optional(), updatedAt: isoDateTime.optional(), diff --git a/src/lib/validations/inbound-documents.ts b/src/lib/validations/inbound-documents.ts index 669c55f91..8e7d6f7bd 100644 --- a/src/lib/validations/inbound-documents.ts +++ b/src/lib/validations/inbound-documents.ts @@ -441,6 +441,12 @@ export interface InboundDocumentDto { * shows its kind icon instead of fetching a thumbnail that 404s. */ hasThumbnail: boolean; + /** + * v1.39.2 (#1038) — the system an imported document came from, or null for + * one added in HealthLog. Paired with `sourceId`, its id over there. + */ + sourceSystem: DocumentSourceSystemValue | null; + sourceId: string | null; createdAt: string; updatedAt: string; } @@ -669,13 +675,59 @@ const vaccinationIdList = z * becoming a second trip. No `userId` field; it is always narrowed from the * session. */ -export const documentCreateSchema = z.object({ - title: z.string().trim().min(1).max(DOCUMENT_TITLE_MAX).optional(), - kind: z.enum(INBOUND_DOCUMENT_KINDS).optional(), - documentDate: isoDateString.optional(), - episodeIds: episodeIdList.optional(), - encounterIds: encounterIdList.optional(), -}); +/** + * v1.39.2 (#1038) — the systems an imported document can say it came from. + * Closed on purpose: the detail sheet names each one, and a free string here + * would be a label the person never chose rendered as if HealthLog vouched for + * it. OTHER covers any importer that is neither. + */ +export const DOCUMENT_SOURCE_SYSTEMS = ["PAPERLESS", "PAPRA", "OTHER"] as const; +export type DocumentSourceSystemValue = + (typeof DOCUMENT_SOURCE_SYSTEMS)[number]; + +/** Narrow a stored `sourceSystem` string to the closed set (null otherwise). */ +export function toDocumentSourceSystem( + value: string | null | undefined, +): DocumentSourceSystemValue | null { + return (DOCUMENT_SOURCE_SYSTEMS as readonly string[]).includes(value ?? "") + ? (value as DocumentSourceSystemValue) + : null; +} + +/** Max length of an imported document's id in its source system. */ +export const DOCUMENT_SOURCE_ID_MAX = 128; + +export const documentCreateSchema = z + .object({ + title: z.string().trim().min(1).max(DOCUMENT_TITLE_MAX).optional(), + kind: z.enum(INBOUND_DOCUMENT_KINDS).optional(), + documentDate: isoDateString.optional(), + episodeIds: episodeIdList.optional(), + encounterIds: encounterIdList.optional(), + /** Where an imported document came from (provenance, shown on detail). */ + sourceSystem: z.enum(DOCUMENT_SOURCE_SYSTEMS).optional(), + /** + * The document's id in `sourceSystem`. Printable ASCII only: it is a key, + * not prose, and it is echoed on the detail sheet. + */ + sourceId: z + .string() + .trim() + .min(1) + .max(DOCUMENT_SOURCE_ID_MAX) + .regex(/^[\x21-\x7e]+$/u, "Expected printable characters without spaces") + .optional(), + /** + * `defer` holds back automatic AI reading for this upload: the thumbnail + * and a local text index still run, the summary and the lab staging do + * not. Absent means today's behaviour. + */ + aiRead: z.enum(["defer"]).optional(), + }) + .refine((v) => v.sourceId === undefined || v.sourceSystem !== undefined, { + path: ["sourceSystem"], + message: "sourceId needs a sourceSystem", + }); export type DocumentCreateInput = z.infer; diff --git a/src/lib/validations/tokens.ts b/src/lib/validations/tokens.ts index 5d475e499..d6e2ae03f 100644 --- a/src/lib/validations/tokens.ts +++ b/src/lib/validations/tokens.ts @@ -24,3 +24,13 @@ export const createMeasurementTokenSchema = z.object({ export type CreateMeasurementTokenInput = z.infer< typeof createMeasurementTokenSchema >; + +/** + * Body of `POST /api/tokens/documents` — mint a Bearer that can upload + * documents and nothing else. The same shape as the measurement mint and for + * the same reasons: no scope field, a bounded lifetime. + */ +export const createDocumentTokenSchema = z.object({ + name: z.string().min(1, "Name required").max(100), + expiresInDays: z.number().int().min(1).max(365).optional(), +}); diff --git a/tests/integration/backup-round-trip.test.ts b/tests/integration/backup-round-trip.test.ts index 2a04eddb3..e262a746a 100644 --- a/tests/integration/backup-round-trip.test.ts +++ b/tests/integration/backup-round-trip.test.ts @@ -823,6 +823,10 @@ async function seedEveryTwoEndedModel(prisma: PrismaClient): Promise { byteSize: documentBytes.byteLength, contentEncrypted, contentCodec: "binary2", + // Imported (#1038): the source key has to survive the trip, or a + // re-run of the importer after a restore stores the page twice. + sourceSystem: "PAPERLESS", + sourceId: "4711", }, }); @@ -1505,6 +1509,10 @@ describe("every model the plan claims two-ended survives a real restore", () => const vaultDocument = await prisma.inboundDocument.findFirstOrThrow({ where: { userId: OWNER_ID }, }); + expect({ + sourceSystem: vaultDocument.sourceSystem, + sourceId: vaultDocument.sourceId, + }).toEqual({ sourceSystem: "PAPERLESS", sourceId: "4711" }); const threads = await prisma.coachConversation.findMany({ where: { userId: OWNER_ID }, orderBy: { createdAt: "asc" }, diff --git a/tests/integration/documents-import.test.ts b/tests/integration/documents-import.test.ts new file mode 100644 index 000000000..83f8d830d --- /dev/null +++ b/tests/integration/documents-import.test.ts @@ -0,0 +1,461 @@ +/** + * Document import (#1038) against a real Postgres, the real resolver and the + * real upload route — no mocked Prisma, no mocked `requireAuth`. + * + * 1. A `documents:write` token minted from a cookie session uploads and gets + * a receipt; a re-send is a duplicate; after the owner deletes the + * document a re-send is answered "deleted" and stores nothing; after the + * purge the ledger keeps answering that. + * 2. The source key's unique index covers tombstones at the database level. + * 3. The token's own bucket 429s with Retry-After without touching the + * owner's; the quota answers 413 with its figures. + * 4. The token reaches no other route: every other vault leg, the mint, and + * the measurement write refuse it. + * 5. With the vault module off the token gets 403 `module.disabled`. + */ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { NextRequest } from "next/server"; + +process.env.ENCRYPTION_KEY ??= + "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"; +process.env.API_TOKEN_HMAC_KEY ??= + "test-hmac-key-documents-import-32-bytes-min-0987654321abcd"; + +import { cookieJar, headerJar } from "./mock-next-headers"; +import { getPrismaClient, truncateAllTables } from "./setup"; + +vi.mock("next/headers", async () => { + const { cookieJar, headerJar } = await import("./mock-next-headers"); + return { + headers: vi.fn(async () => ({ + get: (name: string) => headerJar.get(name.toLowerCase()) ?? null, + })), + cookies: vi.fn(async () => ({ + get: (name: string) => { + const value = cookieJar.get(name); + return value ? { name, value } : undefined; + }, + set: (name: string, value: string) => { + cookieJar.set(name, value); + }, + delete: (name: string) => { + cookieJar.delete(name); + }, + })), + }; +}); + +vi.mock("@/lib/db-compat", () => ({ + ensureDbCompatibility: vi.fn().mockResolvedValue(undefined), +})); + +const PNG_1X1 = Buffer.from( + "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR4nGNgYGBgAAAABQABXvMqOgAAAABJRU5ErkJggg==", + "base64", +); +const pdf = (tag: string) => + Buffer.from(`%PDF-1.7\n% ${tag}\n1 0 obj\n<<>>\nendobj\n%%EOF\n`); + +let userId = ""; +let sessionId = ""; +let token = ""; + +async function seedUser(modules: Record) { + const prisma = getPrismaClient(); + const user = await prisma.user.create({ + data: { + username: "importer", + email: "importer@example.test", + role: "USER", + timezone: "UTC", + modulePreferencesJson: modules, + }, + }); + const session = await prisma.session.create({ + data: { userId: user.id, expiresAt: new Date(Date.now() + 600_000) }, + }); + userId = user.id; + sessionId = session.id; +} + +function asCookie() { + headerJar.clear(); + cookieJar.set("healthlog_session", sessionId); +} + +function asToken() { + cookieJar.clear(); + headerJar.set("authorization", `Bearer ${token}`); +} + +async function mintDocumentToken(): Promise { + asCookie(); + const { POST } = await import("@/app/api/tokens/documents/route"); + const res = await POST( + new NextRequest("https://health.example/api/tokens/documents", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ name: "Paperless workflow" }), + } as never), + ); + expect(res.status).toBe(201); + return (await res.json()).data.token as string; +} + +function upload( + bytes: Buffer, + fields: Record = {}, + filename = "letter.pdf", +): Request { + const payload = new Uint8Array(bytes.byteLength); + payload.set(bytes); + const form = new FormData(); + form.append("file", new Blob([payload]), filename); + for (const [k, v] of Object.entries(fields)) form.append(k, v); + const headers: Record = {}; + const auth = headerJar.get("authorization"); + if (auth) headers.authorization = auth; + return new Request("http://localhost/api/documents/inbound", { + method: "POST", + body: form, + headers, + }); +} + +async function post(r: Request): Promise { + const route = await import("@/app/api/documents/inbound/route"); + return (route.POST as unknown as (r: Request) => Promise)(r); +} + +type Ctx = { params: Promise<{ id: string }> }; +const ctx = (id: string): Ctx => ({ params: Promise.resolve({ id }) }); + +beforeEach(async () => { + await truncateAllTables(getPrismaClient()); + cookieJar.clear(); + headerJar.clear(); + delete process.env.DOCUMENT_UPLOAD_LIMIT_PER_HOUR; +}); + +afterEach(() => { + delete process.env.DOCUMENT_UPLOAD_LIMIT_PER_HOUR; +}); + +describe("document import through a documents:write token", () => { + it("201 → 200 duplicate → delete → 200 deleted, no new row → purge → still deleted", async () => { + await seedUser({ inboundDocuments: true }); + token = await mintDocumentToken(); + const prisma = getPrismaClient(); + + // The mint wrote exactly one narrow scope. + const row = await prisma.apiToken.findFirstOrThrow({ where: { userId } }); + expect(row.permissions).toEqual(["documents:write"]); + + const fields = { + title: "Befund Kardiologie", + documentDate: "2019-03-05", + sourceSystem: "PAPERLESS", + sourceId: "412", + aiRead: "defer", + }; + + asToken(); + const first = await post(upload(pdf("a"), fields)); + expect(first.status).toBe(201); + const receipt = await first.json(); + // A receipt, not the row: no title, filename, kind or links. + expect(Object.keys(receipt.data).sort()).toEqual(["duplicate", "id"]); + expect(receipt.data.duplicate).toBe(false); + const id = receipt.data.id as string; + + const stored = await prisma.inboundDocument.findUniqueOrThrow({ + where: { id }, + omit: { contentEncrypted: true }, + }); + expect(stored).toMatchObject({ + userId, + title: "Befund Kardiologie", + sourceSystem: "PAPERLESS", + sourceId: "412", + summaryState: "NONE", + }); + expect(stored.documentDate?.toISOString().slice(0, 10)).toBe("2019-03-05"); + + // Re-send: same source key → duplicate, even with different bytes. + const again = await post(upload(pdf("a-rescanned"), fields)); + expect(again.status).toBe(200); + expect((await again.json()).data).toEqual({ id, duplicate: true }); + + // The owner sees where it came from. + asCookie(); + const byId = await import("@/app/api/documents/inbound/[id]/route"); + const detail = await ( + byId.GET as unknown as (r: Request, c: Ctx) => Promise + )(new Request(`http://localhost/api/documents/inbound/${id}`), ctx(id)); + expect(detail.status).toBe(200); + expect((await detail.json()).data).toMatchObject({ + sourceSystem: "PAPERLESS", + sourceId: "412", + }); + + // The owner deletes it (tombstone). + const del = await ( + byId.DELETE as unknown as (r: Request, c: Ctx) => Promise + )( + new Request(`http://localhost/api/documents/inbound/${id}`, { + method: "DELETE", + }), + ctx(id), + ); + expect(del.status).toBe(200); + + // A re-run of the importer does not bring it back. + asToken(); + const afterDelete = await post(upload(pdf("a"), fields)); + expect(afterDelete.status).toBe(200); + const deletedBody = await afterDelete.json(); + expect(deletedBody.data).toEqual({ id, duplicate: true, deleted: true }); + expect(deletedBody.meta).toEqual({ duplicate: true, deleted: true }); + expect(await prisma.inboundDocument.count({ where: { userId } })).toBe(1); + + // Past the grace window the purge takes the row and keeps the key. + const { purgeExpiredDocumentTombstones } = + await import("@/lib/jobs/document-purge"); + const purged = await purgeExpiredDocumentTombstones( + prisma as never, + new Date(Date.now() + 31 * 86_400_000), + ); + expect(purged).toBe(1); + expect(await prisma.inboundDocument.count({ where: { userId } })).toBe(0); + expect( + await prisma.documentImportKey.findMany({ + where: { userId }, + select: { sourceSystem: true, sourceId: true }, + }), + ).toEqual([{ sourceSystem: "PAPERLESS", sourceId: "412" }]); + + const afterPurge = await post(upload(pdf("a"), fields)); + expect(afterPurge.status).toBe(200); + expect((await afterPurge.json()).data).toEqual({ + id: null, + duplicate: true, + deleted: true, + }); + expect(await prisma.inboundDocument.count({ where: { userId } })).toBe(0); + + // A different source key is a new document. + const other = await post(upload(pdf("b"), { ...fields, sourceId: "413" })); + expect(other.status).toBe(201); + }); + + it("the source-key index holds tombstones at the database level", async () => { + await seedUser({ inboundDocuments: true }); + const prisma = getPrismaClient(); + const base = { + userId, + mimeType: "application/pdf", + byteSize: 1, + contentEncrypted: new Uint8Array([1]), + sourceSystem: "PAPRA", + sourceId: "doc_1", + }; + await prisma.inboundDocument.create({ + data: { ...base, deletedAt: new Date() }, + }); + await expect( + prisma.inboundDocument.create({ data: base }), + ).rejects.toMatchObject({ code: "P2002" }); + // Rows without a source key are not constrained by it. + await prisma.inboundDocument.create({ + data: { ...base, sourceSystem: null, sourceId: null }, + }); + await prisma.inboundDocument.create({ + data: { ...base, sourceSystem: null, sourceId: null }, + }); + }); + + it("429s the token on its own bucket with Retry-After, and the owner's uploads still pass", async () => { + await seedUser({ inboundDocuments: true }); + token = await mintDocumentToken(); + process.env.DOCUMENT_UPLOAD_LIMIT_PER_HOUR = "1"; + + asToken(); + expect((await post(upload(pdf("one")))).status).toBe(201); + const limited = await post(upload(pdf("two"))); + expect(limited.status).toBe(429); + expect(Number(limited.headers.get("retry-after"))).toBeGreaterThan(0); + expect((await limited.json()).meta.errorCode).toBe( + "documents.inbound.rateLimited", + ); + + asCookie(); + const own = await post(upload(pdf("three"))); + expect(own.status).toBe(201); + // The cookie caller still gets the full row. + expect((await own.json()).data.title).toBeNull(); + }); + + it("413s past the quota with the figures, storing nothing", async () => { + await seedUser({ inboundDocuments: true }); + token = await mintDocumentToken(); + await getPrismaClient().user.update({ + where: { id: userId }, + data: { documentQuotaBytes: BigInt(10) }, + }); + asToken(); + const res = await post(upload(PNG_1X1, {}, "x.png")); + expect(res.status).toBe(413); + const body = await res.json(); + expect(body.meta).toMatchObject({ + reason: "quotaExceeded", + quotaBytes: 10, + }); + expect( + await getPrismaClient().inboundDocument.count({ where: { userId } }), + ).toBe(0); + }); + + it("403 module.disabled while the vault is off", async () => { + await seedUser({ inboundDocuments: false }); + token = await mintDocumentToken(); + asToken(); + const res = await post(upload(pdf("x"))); + expect(res.status).toBe(403); + expect((await res.json()).meta.errorCode).toBe("module.disabled"); + }); +}); + +describe("a documents:write token reaches nothing else", () => { + it("is refused by every other vault leg, the mints and the measurement write", async () => { + await seedUser({ inboundDocuments: true }); + token = await mintDocumentToken(); + asToken(); + const created = await post(upload(pdf("seed"))); + expect(created.status).toBe(201); + const id = (await created.json()).data.id as string; + + const auth = { authorization: `Bearer ${token}` }; + const json = (body: unknown) => ({ + headers: { ...auth, "content-type": "application/json" }, + body: JSON.stringify(body), + }); + const u = (path: string) => `http://localhost${path}`; + type Call = [string, () => Promise]; + type Mod = Record; + const withId = async ( + route: Mod, + mod: string, + method: string, + init: RequestInit = { headers: auth }, + ): Promise => + (route[method] as (r: Request, c: Ctx) => Promise)( + new NextRequest(u(`/api/documents/inbound/${id}${mod}`), { + method, + ...init, + } as never), + ctx(id), + ); + const noId = async ( + route: Mod, + mod: string, + method: string, + init: RequestInit = { headers: auth }, + ): Promise => + (route[method] as (r: Request) => Promise)( + new NextRequest(u(`/api/documents/inbound${mod}`), { + method, + ...init, + } as never), + ); + const r = { + list: await import("@/app/api/documents/inbound/route"), + usage: await import("@/app/api/documents/inbound/usage/route"), + capability: await import("@/app/api/documents/inbound/capability/route"), + reindex: await import("@/app/api/documents/inbound/reindex/route"), + bulk: await import("@/app/api/documents/inbound/bulk/route"), + byId: await import("@/app/api/documents/inbound/[id]/route"), + original: await import("@/app/api/documents/inbound/[id]/original/route"), + thumbnail: + await import("@/app/api/documents/inbound/[id]/thumbnail/route"), + restore: await import("@/app/api/documents/inbound/[id]/restore/route"), + extract: await import("@/app/api/documents/inbound/[id]/extract/route"), + index: await import("@/app/api/documents/inbound/[id]/index/route"), + summary: await import("@/app/api/documents/inbound/[id]/summary/route"), + suggest: await import("@/app/api/documents/inbound/[id]/suggest/route"), + chat: await import("@/app/api/documents/inbound/[id]/chat/route"), + confirm: await import("@/app/api/documents/inbound/[id]/confirm/route"), + } as unknown as Record; + + const calls: Call[] = [ + ["GET list", () => noId(r.list, "", "GET")], + ["GET usage", () => noId(r.usage, "/usage", "GET")], + ["GET capability", () => noId(r.capability, "/capability", "GET")], + ["POST reindex", () => noId(r.reindex, "/reindex", "POST")], + [ + "POST bulk", + () => + noId(r.bulk, "/bulk", "POST", json({ action: "delete", ids: [id] })), + ], + ["GET detail", () => withId(r.byId, "", "GET")], + ["PATCH detail", () => withId(r.byId, "", "PATCH", json({ title: "x" }))], + ["DELETE detail", () => withId(r.byId, "", "DELETE")], + ["GET original", () => withId(r.original, "/original", "GET")], + ["GET thumbnail", () => withId(r.thumbnail, "/thumbnail", "GET")], + ["POST restore", () => withId(r.restore, "/restore", "POST")], + ["POST extract", () => withId(r.extract, "/extract", "POST")], + ["POST index", () => withId(r.index, "/index", "POST")], + ["POST summary", () => withId(r.summary, "/summary", "POST")], + ["POST suggest", () => withId(r.suggest, "/suggest", "POST")], + ["GET chat", () => withId(r.chat, "/chat", "GET")], + [ + "POST confirm", + () => withId(r.confirm, "/confirm", "POST", json({ decisions: [] })), + ], + [ + "POST documents mint", + async () => { + const { POST } = await import("@/app/api/tokens/documents/route"); + return POST( + new NextRequest(u("/api/tokens/documents"), { + method: "POST", + ...json({ name: "second" }), + } as never), + ); + }, + ], + [ + "POST measurements", + async () => { + const { POST } = await import("@/app/api/measurements/route"); + return (POST as unknown as (r: Request) => Promise)( + new NextRequest(u("/api/measurements"), { + method: "POST", + ...json({ type: "WEIGHT", value: 80, measuredAt: new Date() }), + } as never), + ); + }, + ], + ]; + + const outcomes: Record = {}; + for (const [name, call] of calls) { + outcomes[name] = (await call()).status; + } + // Every leg refuses: 403 on a scope decision, 401 on the cookie-only mint. + for (const [name, status] of Object.entries(outcomes)) { + expect({ name, refused: status === 401 || status === 403 }).toEqual({ + name, + refused: true, + }); + } + expect(outcomes["POST documents mint"]).toBe(401); + + // And none of them changed the document. + const row = await getPrismaClient().inboundDocument.findUniqueOrThrow({ + where: { id }, + omit: { contentEncrypted: true }, + }); + expect(row.deletedAt).toBeNull(); + expect(row.title).toBeNull(); + }); +});