diff --git a/.env.production.example b/.env.production.example index fe3144723..edb068b76 100644 --- a/.env.production.example +++ b/.env.production.example @@ -290,6 +290,14 @@ API_TOKEN_HMAC_KEY="" # docs/integrations/ai-providers.md. # DOCUMENT_AI_LIMIT_PER_HOUR="" +# Hourly upload ceiling for each document-upload token (optional). Applies +# only to uploads made with a token minted under Settings > API > Document +# tokens: a Paperless workflow or the import script. Default 120, clamped to +# 1-1000. Uploads from the web app and the phone keep their own bucket. Raise +# it to import a large archive faster; the per-user storage quota still +# applies. See docs/integrations/document-import.md. +# DOCUMENT_UPLOAD_LIMIT_PER_HOUR="" + # Per-user hourly ceiling for lab-report scans (optional). The photo/PDF scan # and the browser-OCR text scan share one bucket per user. Default 6, clamped # to 1-1000. Raise it when users scan a stack of reports at once against a diff --git a/docker-compose.yml b/docker-compose.yml index 4e0897edb..0a0adc15d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -290,6 +290,12 @@ x-healthlog-env: &healthlog_env # (the `environment:` block is a whitelist). See # docs/integrations/ai-providers.md. DOCUMENT_AI_LIMIT_PER_HOUR: "${DOCUMENT_AI_LIMIT_PER_HOUR:-}" + # Hourly upload ceiling for each narrow document-upload token (a Paperless + # workflow, the import script; optional; default 120, clamped 1-1000). + # Uploads from the web app and the phone keep their own bucket. Listed here + # so an operator value in .env reaches the container. See + # docs/integrations/document-import.md. + DOCUMENT_UPLOAD_LIMIT_PER_HOUR: "${DOCUMENT_UPLOAD_LIMIT_PER_HOUR:-}" # Per-user hourly ceiling for lab-report scans (photo/PDF and browser-OCR # text share one bucket; optional; default 6, clamped 1-1000). A scan that # fails before the provider is called hands its slot back. Listed here so an diff --git a/docs/api/openapi.yaml b/docs/api/openapi.yaml index 8e942b3e6..7f26349f3 100644 --- a/docs/api/openapi.yaml +++ b/docs/api/openapi.yaml @@ -2807,10 +2807,11 @@ paths: is no GENERIC mint any more — the POST at this exact path issued `["medication:ingest"]` and was removed, because that scope reached no ingest route while the pre-fail-closed default let it reach everything else. The credentials that work are minted where they are scoped: the per-medication API-endpoint toggle, `POST - /api/mcp/tokens` for a connector, and the sibling `POST /api/tokens/measurements` for third-party measurement - ingest. And unlike the revoke, this read is NOT gated on the operator's instance-wide API switch: that switch - governs the surfaces a token is for, not a token's ability to authenticate, so tokens stay live while it is off - and their owner has to be able to see them. + /api/mcp/tokens` for a connector, the sibling `POST /api/tokens/measurements` for third-party measurement + ingest, and `POST /api/tokens/documents` for document upload from another system. And unlike the revoke, this + read is NOT gated on the operator's instance-wide API switch: that switch governs the surfaces a token is for, + not a token's ability to authenticate, so tokens stay live while it is off and their owner has to be able to see + them. responses: "200": description: The caller's tokens, newest first. @@ -2916,6 +2917,91 @@ paths: application/json: schema: $ref: "#/components/schemas/ErrorEnvelope" + /api/tokens/documents: + post: + tags: + - Auth + summary: Mint a document-upload token + description: >- + Mints a Bearer scoped to exactly `documents:write` and audits the mint. THE RESPONSE CARRIES THE RAW TOKEN, + once; it is stored as an HMAC and no path re-reveals it. + + + **What it can do.** `POST /api/documents/inbound` on its owner's own record, and nothing else. It is the door + another document system pushes through: a Paperless-ngx workflow posting a newly tagged document, or an import + script copying an archive. Uploads through it draw on a bucket of their own, keyed on the token (default 120 an + hour, operator-tunable via `DOCUMENT_UPLOAD_LIMIT_PER_HOUR`), and get back a receipt (`id`, `duplicate`, and + `deleted` when the source key belongs to a document the owner deleted) rather than the stored row. + + + **What it cannot do.** List, read, download, preview, edit, delete or bulk-act on a document, reach the + document-AI routes, act on a shared record, or mint another token. Every other route names no scope and refuses + it 403. + + + Minting requires a COOKIE SESSION, for the reason the measurement mint gives: a day-lived native access token + must not be able to leave behind a year-lived credential. Gated by the operator's instance-wide API switch. Body + capped at 16 KiB; 10 mints per user per minute, and at most 10 live document tokens held at once. Tokens appear + in `GET /api/tokens` and are revoked at `DELETE /api/tokens/{id}`. + requestBody: + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/CreateDocumentTokenRequest" + responses: + "201": + description: Token minted. The `token` field is the only copy of the secret. + content: + application/json: + schema: + $ref: "#/components/schemas/CreateDocumentTokenEnvelope" + "400": + description: Body is not parseable JSON. + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorEnvelope" + "401": *a1 + "403": + description: The operator has switched the API off instance-wide. Nothing was minted. A Bearer caller is refused 401 + before the switch is read. + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorEnvelope" + "409": + description: The account already holds 10 live document tokens (`meta.errorCode` = `tokens.documents.ceiling_reached`). + Nothing was minted. Revoke one at `DELETE /api/tokens/{id}`; only live tokens carrying `documents:write` + count. + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorEnvelope" + "413": + description: Body exceeds 16 KiB. + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorEnvelope" + "415": + description: Content-Type is not `application/json`. + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorEnvelope" + "422": + description: Validation failed. The envelope carries every offending issue. + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorEnvelope" + "429": + description: More than 10 mints from this account in a minute. Nothing was minted. + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorEnvelope" /api/tokens/{id}: delete: tags: @@ -15572,18 +15658,36 @@ paths: tags: - Documents summary: Store a document (no extraction) - description: "STORE-ONLY upload. Stores the raw document ENCRYPTED at rest with `status: STORED` and runs NO extraction - — provider-free, no AI consent / budget / egress. A file is always filable, even with no document-scan provider + description: >- + STORE-ONLY upload. Stores the raw document ENCRYPTED at rest with `status: STORED` and runs NO extraction — + provider-free, no AI consent / budget / egress. A file is always filable, even with no document-scan provider configured. `multipart/form-data`: a `file` plus optional `title`, `kind`, `documentDate` (YYYY-MM-DD), and repeated `episodeIds` form fields (pre-link to the caller's illness/condition episodes). Accepted types (magic-byte sniffed, never the wire Content-Type): PDF/JPEG/PNG/WebP/GIF render inline; Office (docx/xlsx/pptx/doc/xls/ppt), text/CSV/Markdown/RTF, TIFF, HEIC/HEIF, XML/JSON are stored verbatim and served download-only. HEIC is stored as-is but attachment-only — prefer transcoding to JPEG client-side for inline - preview parity. Error contract: `413` with `meta.reason = \"fileTooLarge\"` (+ `maxFileBytes`) or - `\"quotaExceeded\"` (+ `quotaBytes`, `usedBytes`); `415` with `meta.reason = \"unsupportedType\"`. A same-user + preview parity. Error contract: `413` with `meta.reason = "fileTooLarge"` (+ `maxFileBytes`) or + `"quotaExceeded"` (+ `quotaBytes`, `usedBytes`); `415` with `meta.reason = "unsupportedType"`. A same-user duplicate (same bytes) returns 200 + `meta.duplicate: true` with the existing row — not an error. `Idempotency-Key` honoured. Read `GET /api/documents/inbound/usage` for the effective limits before offering an - upload. AI extraction is a separate opt-in action — see `POST /api/documents/inbound/{id}/extract`." + upload. AI extraction is a separate opt-in action — see `POST /api/documents/inbound/{id}/extract`. + + + **Importing from another system (v1.39.2).** Optional `sourceSystem` (`PAPERLESS`, `PAPRA`, `OTHER`) and + `sourceId` (its id there, printable, up to 128 characters; needs `sourceSystem`) key the upload. A re-send with + a known key is a duplicate: 200 with the live row (`meta.duplicate: true`), or, when the owner DELETED that + document, 200 with `data.deleted: true` and `meta.deleted: true` and nothing stored; this holds after the 30-day + purge too. `aiRead=defer` holds back automatic AI reading for this upload: the thumbnail and a local text index + still run, the summary and lab staging do not, so the document waits for the person to read it deliberately. + Absent, behaviour is unchanged. + + + **Narrow token.** Also reachable with a `documents:write` Bearer (minted at `POST /api/tokens/documents`), and + it is the only route that scope reaches. Such a caller draws on a bucket of its own, keyed on the token (default + 120 an hour, `DOCUMENT_UPLOAD_LIMIT_PER_HOUR`, clamped 1-1000; the 429 carries `Retry-After` and + `X-RateLimit-*`), and gets a `DocumentUploadReceipt` instead of the stored row. A cookie or wildcard caller + keeps the 60-an-hour per-user bucket and the full row. With the vault module off the answer is 403 + `module.disabled` for every caller. parameters: - *a6 requestBody: @@ -15628,22 +15732,41 @@ paths: type: array items: type: string + sourceSystem: + description: Optional. The system an imported document comes from. Shown on the detail view. + type: string + enum: + - PAPERLESS + - PAPRA + - OTHER + sourceId: + description: Optional. The document's id in `sourceSystem`; requires it. Unique per account across live and deleted + documents. + type: string + maxLength: 128 + aiRead: + description: Optional. `defer` holds back automatic AI reading for this upload (local index and thumbnail only). + type: string + enum: + - defer required: - file responses: "200": - description: "Duplicate — the same bytes are already stored: the existing row, with `meta.duplicate: true` at the - envelope level." + description: "Duplicate — the same bytes, or the same source key, are already stored: the existing row (a + `DocumentUploadReceipt` for a `documents:write` token), with `meta.duplicate: true` at the envelope level. + When the source key belongs to a document the owner deleted: a receipt with `deleted: true` and + `meta.deleted: true`, nothing stored." content: application/json: schema: - $ref: "#/components/schemas/InboundDocumentEnvelope" + $ref: "#/components/schemas/DocumentUploadEnvelope" "201": - description: The stored document. + description: The stored document (a `DocumentUploadReceipt` for a `documents:write` token). content: application/json: schema: - $ref: "#/components/schemas/InboundDocumentEnvelope" + $ref: "#/components/schemas/DocumentUploadEnvelope" "401": *a1 "409": *a7 "413": @@ -21981,6 +22104,21 @@ components: permission array is a literal, so no request can widen it. `name` is what the token is listed under — name it after the thing you are pasting it into, because that is what you will be reading when you decide whether to revoke it. `expiresInDays` defaults to 365." + CreateDocumentTokenRequest: + type: object + properties: + name: + type: string + minLength: 1 + maxLength: 100 + expiresInDays: + type: integer + minimum: 1 + maximum: 365 + required: + - name + description: "Mint a document-upload Bearer. No `scope` field: the endpoint mints one shape. `name` is what the token is + listed under; name it after the system it is pasted into. `expiresInDays` defaults to 365." RegisterRequest: type: object properties: @@ -30586,16 +30724,17 @@ components: sharing.renounce.not_found, sharing.revoke.already_ended, sharing.revoke.not_found, sharing.session.changed, sharing.switch.invalid, sharing.switch.wrong_transport; source-priority — source-priority.body.invalid_json, source-priority.body.invalid_shape; tokens — - tokens.measurements.ceiling_reached; vaccination — vaccination.booster-invalid, - vaccination.booster-no-antigen, vaccination.booster-out-of-scope, vaccination.encounter-not-found, - vaccination.identity-required, vaccination.invalid, vaccination.link.invalid, vaccination.not-found, - vaccination.practitioner-not-found; workout — workout.batch.invalid, workout.batch.payload_too_large, - workout.batch.too_large; (unprefixed) — about_me_conflict, ai_response_truncated, coach_prefs_conflict, - dashboard_layout_conflict, empty_file, health_score_config_conflict, insights_layout_conflict, - invalid_base_updated_at, json_carries_no_intake_time, json_not_an_array, medication_layout_conflict, - missing_required_columns, modules_conflict, mood_tag_layout_conflict, not_configured, - notification_prefs_conflict, oidc_only, private_origin_not_approved, private_origin_not_grantable, - rate_limited_self, too_many_rows, unreadable_json, upstream_invalid_json, vapid_not_configured" + tokens.documents.ceiling_reached, tokens.measurements.ceiling_reached; vaccination — + vaccination.booster-invalid, vaccination.booster-no-antigen, vaccination.booster-out-of-scope, + vaccination.encounter-not-found, vaccination.identity-required, vaccination.invalid, + vaccination.link.invalid, vaccination.not-found, vaccination.practitioner-not-found; workout — + workout.batch.invalid, workout.batch.payload_too_large, workout.batch.too_large; (unprefixed) — + about_me_conflict, ai_response_truncated, coach_prefs_conflict, dashboard_layout_conflict, empty_file, + health_score_config_conflict, insights_layout_conflict, invalid_base_updated_at, + json_carries_no_intake_time, json_not_an_array, medication_layout_conflict, missing_required_columns, + modules_conflict, mood_tag_layout_conflict, not_configured, notification_prefs_conflict, oidc_only, + private_origin_not_approved, private_origin_not_grantable, rate_limited_self, too_many_rows, + unreadable_json, upstream_invalid_json, vapid_not_configured" type: string capability: description: "On an AI refusal: the capability that is unavailable, one of the keys of `AiCapabilities` (`coach`, @@ -35068,6 +35207,19 @@ components: anyOf: - type: string - type: "null" + sourceSystem: + anyOf: + - type: string + enum: + - PAPERLESS + - PAPRA + - OTHER + - type: "null" + sourceId: + anyOf: + - type: string + maxLength: 128 + - type: "null" summary: anyOf: - type: string @@ -38407,6 +38559,42 @@ components: - expiresAt additionalProperties: false description: The one response that carries a usable measurement-ingest Bearer value. + CreateDocumentTokenEnvelope: + type: object + properties: + data: + $ref: "#/components/schemas/CreateDocumentTokenResponse" + error: + type: "null" + meta: + type: object + properties: + requestId: + type: string + additionalProperties: false + required: + - data + - error + additionalProperties: false + CreateDocumentTokenResponse: + type: object + properties: + token: + type: string + description: The raw `hlk_` Bearer, RETURNED EXACTLY ONCE. It is stored only as an HMAC-SHA256 hash and cannot be + retrieved later. Treat it as a secret. + name: + type: string + expiresAt: + type: string + format: date-time + pattern: ^(?:(?:\d\d[2468][048]|\d\d[13579][26]|\d\d0[48]|[02468][048]00|[13579][26]00)-02-29|\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\d|30)|(?:02)-(?:0[1-9]|1\d|2[0-8])))T(?:(?:[01]\d|2[0-3]):[0-5]\d(?::[0-5]\d(?:\.\d+)?)?(?:Z|([+-](?:[01]\d|2[0-3]):[0-5]\d)))$ + required: + - token + - name + - expiresAt + additionalProperties: false + description: The one response that carries a usable document-upload Bearer value. ApiTokenRevokeEnvelope: type: object properties: @@ -63326,6 +63514,18 @@ components: - type: "null" hasThumbnail: type: boolean + sourceSystem: + anyOf: + - type: string + enum: + - PAPERLESS + - PAPRA + - OTHER + - type: "null" + sourceId: + anyOf: + - type: string + - type: "null" createdAt: type: string updatedAt: @@ -63352,6 +63552,8 @@ components: - lastIndexAttemptAt - lastIndexOutcome - hasThumbnail + - sourceSystem + - sourceId - createdAt - updatedAt additionalProperties: false @@ -63366,7 +63568,9 @@ components: attempt finished (successful or not; null when none ran); `lastIndexOutcome` says why that attempt produced no index, and is null when the attempt succeeded or none ran. `hasThumbnail` is true when a preview thumbnail has been rendered in the background (fetch it from `/api/documents/inbound/{id}/thumbnail`); false means no preview - yet or an unsupported type — show a placeholder. Treat an unknown `kind` value as OTHER when decoding. + yet or an unsupported type — show a placeholder. `sourceSystem` / `sourceId` say where an imported document came + from and its id there (both null for a document added in HealthLog); treat an unknown `sourceSystem` as OTHER + when decoding. Treat an unknown `kind` value as OTHER when decoding. DocumentConditionLink: type: object properties: @@ -63399,11 +63603,13 @@ components: visit-kind ENUM CONSTANT rather than a rendered name: unlike a condition label, which is the person's own words, a visit kind is one of eight closed values whose human name the reader's own bundle owns, so publishing the constant is what lets a German reader and an English one each see it named correctly." - InboundDocumentEnvelope: + DocumentUploadEnvelope: type: object properties: data: - $ref: "#/components/schemas/InboundDocument" + anyOf: + - $ref: "#/components/schemas/InboundDocument" + - $ref: "#/components/schemas/DocumentUploadReceipt" error: type: "null" meta: @@ -63416,6 +63622,27 @@ components: - data - error additionalProperties: false + DocumentUploadReceipt: + type: object + properties: + id: + anyOf: + - type: string + - type: "null" + duplicate: + type: boolean + deleted: + type: boolean + const: true + required: + - id + - duplicate + additionalProperties: false + description: "What an upload answers instead of the stored row. A narrow `documents:write` token always gets this shape + (`duplicate: false` on 201, `true` on 200): a credential that can only add files learns whether a file is + stored, not its title, filename or links. Every caller gets it when the upload's source key belongs to a + document the owner DELETED (`deleted: true`, 200, nothing stored); `id` is the tombstoned row, or null once the + 30-day purge removed it." DocumentAiCapabilityEnvelope: type: object properties: @@ -63700,6 +63927,18 @@ components: - type: "null" hasThumbnail: type: boolean + sourceSystem: + anyOf: + - type: string + enum: + - PAPERLESS + - PAPRA + - OTHER + - type: "null" + sourceId: + anyOf: + - type: string + - type: "null" createdAt: type: string updatedAt: @@ -63755,6 +63994,8 @@ components: - lastIndexAttemptAt - lastIndexOutcome - hasThumbnail + - sourceSystem + - sourceId - createdAt - updatedAt - facts diff --git a/docs/integrations/data-import.md b/docs/integrations/data-import.md index 775093405..5e3d43edd 100644 --- a/docs/integrations/data-import.md +++ b/docs/integrations/data-import.md @@ -19,6 +19,10 @@ HealthLog has four ways to bring data in, all reachable from All paths skip rows that already exist, so re-running an import is safe — it merges rather than duplicating. +Documents (letters, lab reports, scans) from Paperless-ngx or Papra come in +through a document token and an import script instead; see +[`document-import.md`](./document-import.md). + Every import reports what it actually wrote. A file where nothing was written is shown as a failure, a file where some rows landed and some were refused is shown as a partial result, and the refused rows are diff --git a/docs/integrations/document-import.md b/docs/integrations/document-import.md new file mode 100644 index 000000000..51d74e95e --- /dev/null +++ b/docs/integrations/document-import.md @@ -0,0 +1,203 @@ +# Importing documents from Paperless-ngx or Papra + +HealthLog's document vault can take documents from another document system +without you uploading them one by one. There are two ways in, and both use +the same kind of credential: a **document token**. + +- **Paperless-ngx** can send each newly tagged document to HealthLog by + itself, through a workflow. Good for new documents as they arrive. +- The **import script** copies documents from Paperless-ngx or Papra in one + go. Good for the archive you already have, and, run on a schedule, for + keeping Papra in step. + +HealthLog never holds a password or token for Paperless-ngx or Papra. They +push into HealthLog; HealthLog does not reach into them. + +## 1. Turn on Documents and create a document token + +1. In HealthLog, turn on **Documents** under **Settings → Modules** if you + have not already. Uploads are refused while the vault is off. +2. Open **Settings → API & Tokens → Document tokens**, give the token a name (for + example "Paperless workflow"), and select **Create**. +3. Copy the token (it starts with `hlk_`). It is shown once. + +A document token can add documents to your own vault and do nothing else. It +cannot list, open, download, change or delete a document, cannot use the AI +features, and cannot create another token. When it uploads a file it gets +back only whether the file was stored, not the title or anything else about +your vault. You can revoke it at any time under **Settings → API & Tokens → +API Tokens**. It expires after a year unless you revoke it sooner. + +## 2. New documents from Paperless-ngx (workflow) + +In Paperless-ngx, open **Workflows** and add one: + +- **Triggers:** add _Document Added_ and _Document Updated_. On each, set + the filter to "has any of these tags" and pick the tag you use for health + documents, for example `HealthLog`. +- **Action:** _Webhook_. + - URL: `https:///api/documents/inbound` + - **Use parameters** (not JSON), with these parameters: + + | Name | Value | + | -------------- | ---------------------------------------------------- | + | `title` | `{{doc_title}}` | + | `documentDate` | `{{created_year}}-{{created_month}}-{{created_day}}` | + | `sourceSystem` | `PAPERLESS` | + | `sourceId` | `{{doc_id}}` | + + - **Include document:** on. Paperless then sends the original file along + with the parameters, which is exactly what HealthLog expects. + - **Headers:** `Authorization` = `Bearer hlk_…` (your document token). + +Optionally add a parameter `kind` with one of `DOCTOR_REPORT`, +`DISCHARGE_LETTER`, `LAB_RESULT`, `IMAGING`, `PRESCRIPTION`, `REFERRAL`, +`INSURANCE`, `VACCINATION` or `OTHER` if the workflow only ever sends one +kind of document. Without it, documents are filed as _Other_ and you can +change the type in HealthLog. + +Tagging a document again, or editing it in Paperless, fires the workflow +again. That is harmless: HealthLog recognises the Paperless document id and +answers "already stored". If you delete the document in HealthLog, a later +re-send does not bring it back. + +**What the workflow is not for.** Paperless gives up on a webhook after +three quick retries and does not retry at all when a request takes longer +than five seconds. If you tag hundreds of documents at once, most of those +webhooks arrive while HealthLog is still asking them to slow down, and +Paperless drops them. Use the import script below for the existing archive +and let the workflow handle what comes after. + +## 3. The archive: the import script + +The script is a single file, `scripts/import-documents.mjs` in the +HealthLog repository. It needs Node 22 and nothing else, and it runs on any +machine that can reach both systems. Download it: + +```sh +curl -O https://raw.githubusercontent.com/MBombeck/HealthLog/main/scripts/import-documents.mjs +``` + +Tokens go in environment variables, never on the command line: + +```sh +export HEALTHLOG_TOKEN=hlk_… # the document token +export PAPERLESS_TOKEN=… # Paperless: My Profile → API auth token +export PAPRA_TOKEN=… # Papra: an API key with documents:read and tags:read +``` + +**Paperless-ngx:** + +```sh +node import-documents.mjs paperless \ + --paperless-url https://paperless.example \ + --healthlog-url https://health.example \ + --tag HealthLog +``` + +**Papra** (the organization id is in the Papra address bar, +`/organizations//…`): + +```sh +node import-documents.mjs papra \ + --papra-url https://papra.example \ + --papra-org \ + --healthlog-url https://health.example \ + --tag Health +``` + +No Node on the machine? Run it in a throwaway container: + +```sh +docker run --rm -it -v "$PWD:/w" -w /w \ + -e HEALTHLOG_TOKEN -e PAPERLESS_TOKEN \ + node:22-alpine node import-documents.mjs paperless … +``` + +Options: + +- `--tag ` copies only documents with that tag. Recommended; without + it the script copies everything the source token can see. +- `--since YYYY-MM-DD` copies only documents dated on or after that day. +- `--kind LAB_RESULT` files everything as one type. `--kind-map +"Befund=DOCTOR_REPORT,Labor=LAB_RESULT"` picks the type by Paperless + document type or by Papra tag; anything unmatched gets `--kind`, or + _Other_. +- `--dry-run` lists what would be copied, with sizes and a total, and sends + nothing. Worth running first. +- `--ai-read` lets HealthLog read the documents with AI as they arrive (see + below). Off by default. + +What it does: + +- It copies one document at a time and keeps only one file in memory. +- It sends the original file (for Paperless, the original rather than the + archived PDF), the title, the document date, and the source id. +- The document date is the date the source has for the document. When the + source has none, the script uses the day the document was added there, and + lists those documents in its summary so you can correct the date. +- Running it again is safe. HealthLog skips documents it already has, and + documents you deleted in HealthLog stay deleted. An interrupted run simply + picks up where it stopped. +- When HealthLog asks it to slow down, it waits as long as HealthLog says + and carries on. By default a document token may upload 120 documents an + hour, so an archive of a thousand documents takes an evening. Uploads you + make yourself in the web app or on your phone are counted separately and + are not held up. +- At the end it prints how many documents were imported, how many were + already there, how many you had deleted, and every document it skipped + with the reason. It exits with `0` when everything went through, `3` when + some documents were skipped, `1` when it had to stop (for example a + refused token, the vault switched off, or storage full), and `2` for a + mistake in the command. + +### AI reading of imported documents + +If you have turned on automatic AI reading of documents, HealthLog normally +reads each new document with your AI provider as it arrives. For an import +the script holds that back, so a whole archive does not turn into a +thousand AI requests at once. Imported documents still get a preview and +are searchable by their text where the file has a text layer. + +To have documents read later, open one and choose **Read with AI** or +**Generate summary**, or use **Index all for search** in Documents for the +ones that had no text to search. Pass `--ai-read` if you do want every +imported document read as it arrives. + +The Paperless workflow does not hold AI reading back: new documents are +treated like any other upload. + +### Keeping Papra in step + +Papra's webhooks only say that something changed, not what the file is, and +cannot carry a token, so Papra cannot post into HealthLog directly. Run the +script on a schedule instead, for example nightly with cron: + +```cron +15 3 * * * cd /opt/healthlog-import && HEALTHLOG_TOKEN=hlk_… PAPRA_TOKEN=… node import-documents.mjs papra --papra-url https://papra.example --papra-org --healthlog-url https://health.example --tag Health --since 2026-01-01 >> import.log 2>&1 +``` + +Documents already copied are recognised and skipped, so a nightly run only +adds what is new. `--since` keeps each run short. + +## Storage + +HealthLog keeps its own encrypted copy of each document, so an imported +archive takes space in both systems. The preview, text search, lab import, +the doctor report and sharing all need the file itself, which is why +HealthLog does not link to documents stored elsewhere. + +The default storage limit is 1 GB per person, counting deleted documents +until they are removed for good 30 days later. Run the script with +`--dry-run` to see how much an import needs. An admin can raise the limit in +the admin area, for everyone or for one account. If storage runs out during +an import, the script stops and says so; raise the limit and run it again. + +## For operators + +`DOCUMENT_UPLOAD_LIMIT_PER_HOUR` sets how many documents one document token +may upload per hour (default `120`, clamped to 1–1000). It applies to +document tokens only; uploads from the web app and the phone keep their own +limit of 60 per hour per person. Raise it to let a large import finish +faster; the storage limit still applies. The variable is on the +`docker-compose.yml` whitelist, so setting it in `.env` is enough. diff --git a/messages/de.json b/messages/de.json index f80638722..9c1082c04 100644 --- a/messages/de.json +++ b/messages/de.json @@ -6140,7 +6140,14 @@ "scopeNote": "Es kann ausschließlich neue Werte in deine eigene Akte eintragen — deine Daten nicht lesen, nichts ändern oder löschen und kein weiteres Token erstellen. Werte werden als von einem externen Gerät gekennzeichnet, damit du sie später erkennst, und lassen sich jederzeit korrigieren.", "createFailed": "Das Token konnte nicht erstellt werden. Bitte versuche es erneut." }, - "tokenMintMovedDescription": "Tokens werden dort ausgestellt, wo sie gelten: oben für Messwerte, oder im API-Endpoint eines Medikaments für genau dieses Medikament.", + "documentsToken": { + "title": "Dokument-Tokens", + "description": "Erzeuge ein Token, damit Paperless-ngx, Papra oder das Import-Skript Dokumente in deine Ablage legen können.", + "detail": "Das Token wird nur einmal angezeigt.", + "scopeNote": "Es kann ausschließlich neue Dokumente in deine eigene Ablage legen. Es kann nichts auflisten, öffnen, herunterladen, ändern oder löschen und kein weiteres Token erstellen. Zurück bekommt es nur, ob eine Datei gespeichert wurde, nicht wie du sie abgelegt hast.", + "createFailed": "Das Token konnte nicht erstellt werden. Bitte versuche es erneut." + }, + "tokenMintMovedDescription": "Tokens werden dort ausgestellt, wo sie gelten: oben für Messwerte und Dokumente, oder im API-Endpoint eines Medikaments für genau dieses Medikament.", "tokenNamePlaceholder": "Token-Name (z.B. iPhone Shortcut)", "tokenCreated": "Token erstellt — kopiere ihn jetzt! Er wird nicht erneut angezeigt.", "tokenCopy": "Token kopieren", @@ -9984,6 +9991,9 @@ "noConditions": "Mit keiner Erkrankung verknüpft", "previewUnavailable": "Keine Inline-Vorschau für diesen Dateityp", "uploadedMeta": "Hinzugefügt am {date} · {size}", + "importedFrom": "Importiert aus {system}", + "importedFromWithId": "Importiert aus {system} · ID {id}", + "sourceOther": "einem anderen System", "share": "Teilen", "labValuesLink": "{count} Laborwerte ansehen", "visitsLabel": "Gehört zu Termin", diff --git a/messages/en.json b/messages/en.json index ec7b4cbdf..a1be3a8bb 100644 --- a/messages/en.json +++ b/messages/en.json @@ -6140,7 +6140,14 @@ "scopeNote": "It can only add new readings to your own record — it cannot read your data, change or delete anything, or create another token. Readings are marked as coming from an external device, so you can tell them apart later, and you can still correct them yourself.", "createFailed": "The token could not be created. Please try again." }, - "tokenMintMovedDescription": "Tokens are issued where they apply: above for measurements, or in a medication's own API endpoint for that one medication.", + "documentsToken": { + "title": "Document tokens", + "description": "Mint a token so Paperless-ngx, Papra or the import script can add documents to your vault.", + "detail": "The token is shown once.", + "scopeNote": "It can only add new documents to your own vault. It cannot list, open, download, change or delete anything, and it cannot create another token. What it gets back is whether a file was stored, not how you filed it.", + "createFailed": "The token could not be created. Please try again." + }, + "tokenMintMovedDescription": "Tokens are issued where they apply: above for measurements and documents, or in a medication's own API endpoint for that one medication.", "tokenNamePlaceholder": "Token name (e.g., iPhone Shortcut)", "tokenCreated": "Token created — copy it now! It won't be shown again.", "tokenCopy": "Copy token", @@ -9984,6 +9991,9 @@ "noConditions": "Not linked to a condition", "previewUnavailable": "No inline preview for this file type", "uploadedMeta": "Added {date} · {size}", + "importedFrom": "Imported from {system}", + "importedFromWithId": "Imported from {system} · ID {id}", + "sourceOther": "another system", "share": "Share", "labValuesLink": "View {count} lab values", "visitsLabel": "Belongs to visit", diff --git a/messages/es.json b/messages/es.json index b64f58204..b89757989 100644 --- a/messages/es.json +++ b/messages/es.json @@ -6140,7 +6140,14 @@ "scopeNote": "Solo puede añadir mediciones nuevas a tu propio historial: no puede leer tus datos, ni modificar o borrar nada, ni crear otro token. Las mediciones se marcan como procedentes de un dispositivo externo, así las distingues más tarde, y puedes corregirlas igualmente.", "createFailed": "No se pudo crear el token. Inténtalo de nuevo." }, - "tokenMintMovedDescription": "Los tokens se emiten donde se aplican: arriba para las mediciones, o en el endpoint de API de un medicamento para ese medicamento concreto.", + "documentsToken": { + "title": "Tokens de documentos", + "description": "Crea un token para que Paperless-ngx, Papra o el script de importación añadan documentos a tu archivo.", + "detail": "El token se muestra una sola vez.", + "scopeNote": "Solo puede añadir documentos nuevos a tu propio archivo. No puede listar, abrir, descargar, modificar ni borrar nada, ni crear otro token. Lo único que recibe es si un archivo se guardó, no cómo lo archivaste.", + "createFailed": "No se pudo crear el token. Inténtalo de nuevo." + }, + "tokenMintMovedDescription": "Los tokens se emiten donde se aplican: arriba para las mediciones y los documentos, o en el endpoint de API de un medicamento para ese medicamento concreto.", "tokenNamePlaceholder": "Nombre del token (p. ej., Atajo de iPhone)", "tokenCreated": "Token creado — ¡cópialo ahora! No se mostrará de nuevo.", "tokenCopy": "Copiar token", @@ -9984,6 +9991,9 @@ "noConditions": "Sin vincular a una afección", "previewUnavailable": "Sin vista previa integrada para este tipo de archivo", "uploadedMeta": "Añadido el {date} · {size}", + "importedFrom": "Importado de {system}", + "importedFromWithId": "Importado de {system} · ID {id}", + "sourceOther": "otro sistema", "share": "Compartir", "labValuesLink": "Ver {count} valores de laboratorio", "visitsLabel": "Pertenece a la cita", diff --git a/messages/fr.json b/messages/fr.json index a1884c959..2c73fe8c1 100644 --- a/messages/fr.json +++ b/messages/fr.json @@ -6140,7 +6140,14 @@ "scopeNote": "Il peut uniquement ajouter de nouvelles mesures à votre propre dossier : il ne peut pas lire vos données, ni modifier ou supprimer quoi que ce soit, ni créer un autre jeton. Les mesures sont signalées comme provenant d'un appareil externe, ce qui vous permet de les reconnaître plus tard, et vous pouvez toujours les corriger.", "createFailed": "Le jeton n'a pas pu être créé. Veuillez réessayer." }, - "tokenMintMovedDescription": "Les jetons sont émis là où ils s'appliquent : ci-dessus pour les mesures, ou dans le point de terminaison API d'un médicament pour ce seul médicament.", + "documentsToken": { + "title": "Jetons de documents", + "description": "Créez un jeton pour que Paperless-ngx, Papra ou le script d'import ajoutent des documents à votre coffre.", + "detail": "Le jeton n'est affiché qu'une fois.", + "scopeNote": "Il peut uniquement ajouter de nouveaux documents à votre propre coffre. Il ne peut rien lister, ouvrir, télécharger, modifier ou supprimer, ni créer un autre jeton. Il apprend seulement si un fichier a été enregistré, pas comment vous l'avez classé.", + "createFailed": "Le jeton n'a pas pu être créé. Veuillez réessayer." + }, + "tokenMintMovedDescription": "Les jetons sont émis là où ils s'appliquent : ci-dessus pour les mesures et les documents, ou dans le point de terminaison API d'un médicament pour ce seul médicament.", "tokenNamePlaceholder": "Nom du jeton (par exemple Raccourci iPhone)", "tokenCreated": "Jeton créé — copiez-le maintenant ! Il ne sera plus affiché.", "tokenCopy": "Copier le jeton", @@ -9984,6 +9991,9 @@ "noConditions": "Lié à aucune affection", "previewUnavailable": "Pas d'aperçu intégré pour ce type de fichier", "uploadedMeta": "Ajouté le {date} · {size}", + "importedFrom": "Importé depuis {system}", + "importedFromWithId": "Importé depuis {system} · ID {id}", + "sourceOther": "un autre système", "share": "Partager", "labValuesLink": "Voir {count} valeurs de laboratoire", "visitsLabel": "Rattaché au rendez-vous", diff --git a/messages/it.json b/messages/it.json index 23b5b04d4..ded9178bc 100644 --- a/messages/it.json +++ b/messages/it.json @@ -6140,7 +6140,14 @@ "scopeNote": "Può soltanto aggiungere nuove misurazioni alla tua cartella: non può leggere i tuoi dati, modificare o eliminare nulla, né creare un altro token. Le misurazioni vengono contrassegnate come provenienti da un dispositivo esterno, così le riconosci in seguito, e puoi comunque correggerle.", "createFailed": "Non è stato possibile creare il token. Riprova." }, - "tokenMintMovedDescription": "I token vengono emessi dove si applicano: qui sopra per le misurazioni, oppure nell'endpoint API di un farmaco per quel singolo farmaco.", + "documentsToken": { + "title": "Token per documenti", + "description": "Crea un token perché Paperless-ngx, Papra o lo script di importazione aggiungano documenti al tuo archivio.", + "detail": "Il token viene mostrato una sola volta.", + "scopeNote": "Può soltanto aggiungere nuovi documenti al tuo archivio. Non può elencare, aprire, scaricare, modificare o eliminare nulla, né creare un altro token. Riceve solo l'informazione se un file è stato salvato, non come l'hai archiviato.", + "createFailed": "Non è stato possibile creare il token. Riprova." + }, + "tokenMintMovedDescription": "I token vengono emessi dove si applicano: qui sopra per le misurazioni e i documenti, oppure nell'endpoint API di un farmaco per quel singolo farmaco.", "tokenNamePlaceholder": "Nome del token (ad es. Comando rapido iPhone)", "tokenCreated": "Token creato — copialo ora! Non verrà mostrato di nuovo.", "tokenCopy": "Copia token", @@ -9984,6 +9991,9 @@ "noConditions": "Non collegato a una patologia", "previewUnavailable": "Nessuna anteprima integrata per questo tipo di file", "uploadedMeta": "Aggiunto il {date} · {size}", + "importedFrom": "Importato da {system}", + "importedFromWithId": "Importato da {system} · ID {id}", + "sourceOther": "un altro sistema", "share": "Condividi", "labValuesLink": "Vedi {count} valori di laboratorio", "visitsLabel": "Appartiene all'appuntamento", diff --git a/messages/ko.json b/messages/ko.json index fb5db9cc2..b63e830fe 100644 --- a/messages/ko.json +++ b/messages/ko.json @@ -6140,7 +6140,14 @@ "scopeNote": "본인 기록에 새 측정값을 추가하는 것만 할 수 있어요. 데이터를 읽거나, 무언가를 고치거나 지우거나, 다른 토큰을 만들 수는 없어요. 측정값은 외부 기기에서 온 것으로 표시되니 나중에 구분할 수 있고, 언제든 고칠 수 있어요.", "createFailed": "토큰을 만들지 못했어요. 다시 시도해 주세요." }, - "tokenMintMovedDescription": "토큰은 쓰이는 곳에서 발급돼요. 측정값은 위에서, 특정 약은 그 약의 API 엔드포인트에서 만들 수 있어요.", + "documentsToken": { + "title": "문서 토큰", + "description": "Paperless-ngx, Papra 또는 가져오기 스크립트가 문서함에 문서를 추가할 수 있도록 토큰을 만들어요.", + "detail": "토큰은 한 번만 표시돼요.", + "scopeNote": "본인 문서함에 새 문서를 추가하는 것만 할 수 있어요. 목록을 보거나, 열거나, 내려받거나, 고치거나 지울 수 없고, 다른 토큰을 만들 수도 없어요. 파일이 저장됐는지만 알 수 있고, 어떻게 정리했는지는 알 수 없어요.", + "createFailed": "토큰을 만들지 못했어요. 다시 시도해 주세요." + }, + "tokenMintMovedDescription": "토큰은 쓰이는 곳에서 발급돼요. 측정값과 문서는 위에서, 특정 약은 그 약의 API 엔드포인트에서 만들 수 있어요.", "tokenNamePlaceholder": "토큰 이름 (예: iPhone 단축어)", "tokenCreated": "토큰을 만들었어요 — 지금 복사하세요! 다시 보여주지 않아요.", "tokenCopy": "토큰 복사", @@ -9984,6 +9991,9 @@ "noConditions": "질환에 연결되지 않음", "previewUnavailable": "이 파일 형식은 미리보기를 지원하지 않아요", "uploadedMeta": "추가 {date} · {size}", + "importedFrom": "{system}에서 가져옴", + "importedFromWithId": "{system}에서 가져옴 · ID {id}", + "sourceOther": "다른 시스템", "share": "공유", "labValuesLink": "검사 수치 {count}개 보기", "visitsLabel": "연결된 진료", diff --git a/messages/pl.json b/messages/pl.json index cc0535130..3f4f9392c 100644 --- a/messages/pl.json +++ b/messages/pl.json @@ -6140,7 +6140,14 @@ "scopeNote": "Może wyłącznie dodawać nowe pomiary do Twojej dokumentacji — nie odczyta danych, niczego nie zmieni ani nie usunie i nie utworzy kolejnego tokenu. Pomiary są oznaczane jako pochodzące z urządzenia zewnętrznego, dzięki czemu później je rozpoznasz, i nadal możesz je poprawić.", "createFailed": "Nie udało się utworzyć tokenu. Spróbuj ponownie." }, - "tokenMintMovedDescription": "Tokeny wydajemy tam, gdzie działają: powyżej dla pomiarów albo w punkcie końcowym API danego leku dla tego jednego leku.", + "documentsToken": { + "title": "Tokeny dokumentów", + "description": "Utwórz token, aby Paperless-ngx, Papra lub skrypt importu mogły dodawać dokumenty do Twojego archiwum.", + "detail": "Token pokazujemy tylko raz.", + "scopeNote": "Może wyłącznie dodawać nowe dokumenty do Twojego archiwum. Niczego nie wyświetli, nie otworzy, nie pobierze, nie zmieni ani nie usunie i nie utworzy kolejnego tokenu. Dowiaduje się tylko, czy plik został zapisany, a nie to, jak został opisany.", + "createFailed": "Nie udało się utworzyć tokenu. Spróbuj ponownie." + }, + "tokenMintMovedDescription": "Tokeny wydajemy tam, gdzie działają: powyżej dla pomiarów i dokumentów albo w punkcie końcowym API danego leku dla tego jednego leku.", "tokenNamePlaceholder": "Nazwa tokena (np. Skrót iPhone)", "tokenCreated": "Token utworzony — skopiuj go teraz! Nie zostanie pokazany ponownie.", "tokenCopy": "Kopiuj token", @@ -9984,6 +9991,9 @@ "noConditions": "Niepowiązany ze schorzeniem", "previewUnavailable": "Brak podglądu dla tego typu pliku", "uploadedMeta": "Dodano {date} · {size}", + "importedFrom": "Zaimportowano z: {system}", + "importedFromWithId": "Zaimportowano z: {system} · ID {id}", + "sourceOther": "inny system", "share": "Udostępnij", "labValuesLink": "Zobacz {count} wyników laboratoryjnych", "visitsLabel": "Należy do wizyty", diff --git a/prisma/migrations/0353_document_import_source/migration.sql b/prisma/migrations/0353_document_import_source/migration.sql new file mode 100644 index 000000000..65c899381 --- /dev/null +++ b/prisma/migrations/0353_document_import_source/migration.sql @@ -0,0 +1,31 @@ +-- Document import (#1038). Two optional provenance columns on the vault, a +-- per-user unique source key that covers tombstoned rows too, and a small +-- ledger the purge fills so a re-run importer cannot bring back a deleted +-- document after its tombstone is gone. Existing rows keep NULL and are not +-- touched by the index. +ALTER TABLE "inbound_documents" + ADD COLUMN "source_system" VARCHAR(16), + ADD COLUMN "source_id" VARCHAR(128); + +-- Partial unique index (documented in schema.prisma; Prisma cannot express a +-- partial index). No `deleted_at` predicate on purpose: a tombstoned document +-- still owns its source key. +CREATE UNIQUE INDEX "inbound_documents_user_source_key" + ON "inbound_documents" ("user_id", "source_system", "source_id") + WHERE "source_id" IS NOT NULL; + +CREATE TABLE "document_import_keys" ( + "id" TEXT NOT NULL, + "user_id" TEXT NOT NULL, + "source_system" VARCHAR(16) NOT NULL, + "source_id" VARCHAR(128) NOT NULL, + "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + CONSTRAINT "document_import_keys_pkey" PRIMARY KEY ("id") +); + +CREATE UNIQUE INDEX "document_import_keys_user_id_source_system_source_id_key" + ON "document_import_keys" ("user_id", "source_system", "source_id"); + +ALTER TABLE "document_import_keys" + ADD CONSTRAINT "document_import_keys_user_id_fkey" + FOREIGN KEY ("user_id") REFERENCES "users"("id") ON DELETE CASCADE ON UPDATE CASCADE; diff --git a/prisma/schema.prisma b/prisma/schema.prisma index 9cff57b4e..202846026 100644 --- a/prisma/schema.prisma +++ b/prisma/schema.prisma @@ -922,6 +922,9 @@ model User { // (ExtractedFact) and only reach the structured stores on explicit // approval. Opt-in module, off by default. inboundDocuments InboundDocument[] + // v1.39.2 — source keys of imported documents that the tombstone purge + // removed, so a re-run of an importer cannot bring a deleted document back. + documentImportKeys DocumentImportKey[] extractedFacts ExtractedFact[] documentConditionLinks DocumentConditionLink[] documentContentIndexes DocumentContentIndex[] @@ -7938,6 +7941,22 @@ model InboundDocument { /// from "tried and here is why it failed" instead of showing a bare /// not-indexed state for both. lastIndexOutcome String? @map("last_index_outcome") + /// v1.39.2 (#1038) — where an imported document came from: a closed string + /// enum (`DOCUMENT_SOURCE_SYSTEMS` in + /// `src/lib/validations/inbound-documents.ts`): PAPERLESS | PAPRA | OTHER. + /// Null for a document added in HealthLog itself. Sent by the import script + /// and the Paperless workflow recipe; shown on the detail sheet. + sourceSystem String? @map("source_system") @db.VarChar(16) + /// v1.39.2 (#1038) — the document's id in `sourceSystem`. Together with + /// `userId` + `sourceSystem` it is unique across live AND tombstoned rows + /// (raw SQL in migration 0353, not expressible in Prisma): + /// CREATE UNIQUE INDEX inbound_documents_user_source_key + /// ON inbound_documents (user_id, source_system, source_id) + /// WHERE source_id IS NOT NULL; + /// Including tombstones is the point: re-running an importer must not bring + /// back a document the person deleted. The purge carries the key over to + /// `DocumentImportKey` when it removes the row. + sourceId String? @map("source_id") @db.VarChar(128) facts ExtractedFact[] conditionLinks DocumentConditionLink[] @@ -7984,6 +8003,26 @@ model InboundDocument { @@map("inbound_documents") } +/// v1.39.2 (#1038) — the source key of an imported document that the daily +/// tombstone purge has physically removed. While the tombstone exists the key +/// lives on the document row itself; once the purge takes the row, this is the +/// only memory that the person deleted it, so a re-run of the importer answers +/// "deleted" instead of storing the document again. Written by the purge only; +/// read by the upload route's source-key check. Holds no document content. +model DocumentImportKey { + id String @id @default(cuid()) + userId String @map("user_id") + /// Same closed vocabulary as `InboundDocument.sourceSystem`. + sourceSystem String @map("source_system") @db.VarChar(16) + sourceId String @map("source_id") @db.VarChar(128) + createdAt DateTime @default(now()) @map("created_at") + + user User @relation(fields: [userId], references: [id], onDelete: Cascade) + + @@unique([userId, sourceSystem, sourceId]) + @@map("document_import_keys") +} + model ExtractedFact { id String @id @default(cuid()) documentId String @map("document_id") diff --git a/scripts/__tests__/import-documents.test.ts b/scripts/__tests__/import-documents.test.ts new file mode 100644 index 000000000..4a8af6d45 --- /dev/null +++ b/scripts/__tests__/import-documents.test.ts @@ -0,0 +1,531 @@ +/** + * `scripts/import-documents.mjs` against recorded-shape fixtures of the two + * source APIs and a stand-in HealthLog, all on local HTTP servers. + * + * The fixtures are the shapes the script depends on (Paperless-ngx API v10 + * pagination, tag and document-type lookups, `?original=true` downloads; + * Papra's organization-scoped document list, tags and `/file`), so a change + * on either side shows up here as a fixture to refresh rather than as a user + * report. The HealthLog stand-in answers the upload contract the real route + * speaks: 201, 200 duplicate, 200 deleted, 413, 429 with Retry-After, 403 + * `module.disabled`. + */ +import { spawn } from "node:child_process"; +import { createServer, type IncomingMessage, type Server } from "node:http"; +import type { AddressInfo } from "node:net"; +import { join } from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; + +const SCRIPT = join(process.cwd(), "scripts/import-documents.mjs"); +const PDF = Buffer.from("%PDF-1.7\n%fixture\n%%EOF\n"); + +const servers: Server[] = []; +afterEach(async () => { + await Promise.all( + servers.splice(0).map((s) => new Promise((r) => s.close(() => r(null)))), + ); +}); + +type Handler = ( + req: IncomingMessage, + url: URL, + body: Buffer, +) => Promise<{ + status: number; + json?: unknown; + bytes?: Buffer; + headers?: Record; +}>; + +async function serve(handler: Handler): Promise { + const server = createServer(async (req, res) => { + const chunks: Buffer[] = []; + for await (const c of req) chunks.push(c as Buffer); + const url = new URL(req.url ?? "/", "http://localhost"); + const r = await handler(req, url, Buffer.concat(chunks)); + const headers: Record = { ...(r.headers ?? {}) }; + if (r.json !== undefined) headers["content-type"] = "application/json"; + res.writeHead(r.status, headers); + res.end(r.json !== undefined ? JSON.stringify(r.json) : (r.bytes ?? "")); + }); + servers.push(server); + await new Promise((r) => server.listen(0, "127.0.0.1", () => r())); + return `http://127.0.0.1:${(server.address() as AddressInfo).port}`; +} + +function runScript( + args: string[], + env: Record, +): Promise<{ code: number | null; stdout: string; stderr: string }> { + return new Promise((resolve) => { + const child = spawn(process.execPath, [SCRIPT, ...args], { + env: { PATH: process.env.PATH ?? "", ...env }, + }); + let stdout = ""; + let stderr = ""; + child.stdout.on("data", (d) => (stdout += d)); + child.stderr.on("data", (d) => (stderr += d)); + child.on("close", (code) => resolve({ code, stdout, stderr })); + }); +} + +interface Received { + auth: string | undefined; + fields: Record; + filename: string | null; + size: number; +} + +/** + * A HealthLog stand-in. Keys stored documents by source key the way the real + * route does, answers the first upload with a 429, and refuses keys listed as + * deleted or too large. + */ +async function healthlog( + opts: { + deleted?: string[]; + tooLarge?: string[]; + moduleOff?: boolean; + } = {}, +) { + const stored = new Set(); + const received: Received[] = []; + let throttled = false; + const url = await serve(async (req, url, body) => { + if (url.pathname !== "/api/documents/inbound" || req.method !== "POST") { + return { status: 404, json: { data: null, error: "Not found" } }; + } + if (opts.moduleOff) { + return { + status: 403, + json: { + data: null, + error: 'Module "inboundDocuments" is not enabled', + meta: { errorCode: "module.disabled" }, + }, + }; + } + if (!throttled) { + throttled = true; + return { + status: 429, + headers: { "retry-after": "0" }, + json: { + data: null, + error: "Too many uploads.", + meta: { errorCode: "documents.inbound.rateLimited" }, + }, + }; + } + const form = await new Response(body, { + headers: { "content-type": req.headers["content-type"] ?? "" }, + }).formData(); + const fields: Record = {}; + let filename: string | null = null; + let size = 0; + for (const [k, v] of form.entries()) { + if (typeof v === "string") fields[k] = v; + else { + filename = v.name; + size = v.size; + } + } + received.push({ auth: req.headers.authorization, fields, filename, size }); + const key = `${fields.sourceSystem}:${fields.sourceId}`; + if (opts.tooLarge?.includes(key)) { + return { + status: 413, + json: { + data: null, + error: "File is too large.", + meta: { reason: "fileTooLarge", maxFileBytes: 26_214_400 }, + }, + }; + } + if (opts.deleted?.includes(key)) { + return { + status: 200, + json: { + data: { id: null, duplicate: true, deleted: true }, + error: null, + meta: { duplicate: true, deleted: true }, + }, + }; + } + if (stored.has(key)) { + return { + status: 200, + json: { + data: { id: key, duplicate: true }, + error: null, + meta: { duplicate: true }, + }, + }; + } + stored.add(key); + return { + status: 201, + json: { data: { id: key, duplicate: false }, error: null }, + }; + }); + return { url, received }; +} + +/** Paperless-ngx v10 stand-in: two pages of documents, one outside the tag. */ +async function paperless() { + const seen: { path: string; auth?: string; accept?: string }[] = []; + const url = await serve(async (req, url) => { + seen.push({ + path: url.pathname + url.search, + auth: req.headers.authorization, + accept: req.headers.accept, + }); + const base = `http://${req.headers.host}`; + if (url.pathname === "/api/tags/") { + return { + status: 200, + json: { + count: 1, + next: null, + results: [{ id: 7, name: "HealthLog" }], + }, + }; + } + if (url.pathname === "/api/document_types/") { + return { + status: 200, + json: { count: 1, next: null, results: [{ id: 3, name: "Labor" }] }, + }; + } + if (url.pathname === "/api/documents/") { + if (url.searchParams.get("page") === "2") { + return { + status: 200, + json: { + count: 3, + next: null, + results: [ + { + id: 3, + title: "Car insurance", + created: "2024-05-01", + added: "2024-05-02T10:00:00Z", + tags: [9], + document_type: null, + original_file_name: "car.pdf", + }, + ], + }, + }; + } + const next = new URL(url.toString(), base); + next.searchParams.set("page", "2"); + return { + status: 200, + json: { + count: 3, + // Deliberately another origin, as behind a reverse proxy. + next: `https://paperless.internal${next.pathname}${next.search}`, + results: [ + { + id: 1, + title: "Blutbild März", + created: "2024-03-05", + added: "2024-03-06T08:00:00Z", + tags: [7], + document_type: 3, + original_file_name: "blutbild.pdf", + }, + { + id: 2, + title: "Arztbrief", + created: null, + added: "2024-04-01T09:30:00Z", + tags: [7], + document_type: null, + original_file_name: "brief.pdf", + }, + ], + }, + }; + } + const download = /^\/api\/documents\/(\d+)\/download\/$/.exec(url.pathname); + if (download) { + return { + status: 200, + bytes: PDF, + headers: { + "content-type": "application/pdf", + "content-disposition": `attachment; filename="doc-${download[1]}.pdf"`, + }, + }; + } + const metadata = /^\/api\/documents\/(\d+)\/metadata\/$/.exec(url.pathname); + if (metadata) { + return { status: 200, json: { original_size: 1_048_576 } }; + } + return { status: 404, json: { detail: "Not found." } }; + }); + return { url, seen }; +} + +/** Papra stand-in: one organization, a tag, three documents. */ +async function papra() { + const seen: { path: string; auth?: string }[] = []; + const url = await serve(async (req, url) => { + seen.push({ + path: url.pathname + url.search, + auth: req.headers.authorization, + }); + const org = "/api/organizations/org_1"; + if (url.pathname === `${org}/tags`) { + return { + status: 200, + json: { + tags: [ + { id: "tag_h", name: "Health" }, + { id: "tag_l", name: "Lab" }, + ], + }, + }; + } + if (url.pathname === `${org}/documents`) { + return { + status: 200, + json: { + documentsCount: 3, + documents: [ + { + id: "doc_a", + name: "Impfpass.pdf", + originalName: "impfpass.pdf", + originalSize: 2048, + createdAt: "2025-01-10T12:00:00.000Z", + documentDate: "2019-06-01", + tags: [{ id: "tag_h", name: "Health" }], + }, + { + id: "doc_b", + name: "Labor.pdf", + originalName: "labor.pdf", + originalSize: 4096, + createdAt: "2025-02-10T12:00:00.000Z", + tags: [ + { id: "tag_h", name: "Health" }, + { id: "tag_l", name: "Lab" }, + ], + }, + { + id: "doc_old", + name: "Old.pdf", + originalName: "old.pdf", + originalSize: 100, + createdAt: "2015-02-10T12:00:00.000Z", + documentDate: "2015-02-10", + tags: [{ id: "tag_h", name: "Health" }], + }, + ], + }, + }; + } + if ( + /^\/api\/organizations\/org_1\/documents\/[\w]+\/file$/.test(url.pathname) + ) { + return { + status: 200, + bytes: PDF, + headers: { "content-type": "application/pdf" }, + }; + } + return { status: 404, json: { error: "not found" } }; + }); + return { url, seen }; +} + +describe("import-documents.mjs — Paperless-ngx", () => { + it("imports the tagged documents with their title, date, kind and source key", async () => { + const hl = await healthlog(); + const pl = await paperless(); + const env = { HEALTHLOG_TOKEN: "hlk_test", PAPERLESS_TOKEN: "pl_test" }; + const args = [ + "paperless", + "--paperless-url", + pl.url, + "--healthlog-url", + hl.url, + "--tag", + "HealthLog", + "--kind-map", + "Labor=LAB_RESULT", + ]; + const first = await runScript(args, env); + + expect(first.stderr).toBe(""); + expect(first.code).toBe(0); + // The first upload met a 429 and was sent again, not dropped. + expect(hl.received).toHaveLength(2); + expect(hl.received[0]).toMatchObject({ + auth: "Bearer hlk_test", + filename: "blutbild.pdf", + size: PDF.byteLength, + fields: { + title: "Blutbild März", + documentDate: "2024-03-05", + kind: "LAB_RESULT", + sourceSystem: "PAPERLESS", + sourceId: "1", + aiRead: "defer", + }, + }); + // No created date: filed under the day it was added, and said so. + expect(hl.received[1].fields.documentDate).toBe("2024-04-01"); + expect(hl.received[1].fields.kind).toBeUndefined(); + expect(first.stdout).toMatch(/Imported: 2/); + expect(first.stdout).toMatch(/filed under the day it was added/); + expect(first.stdout).toMatch(/without automatic AI reading/); + + // The API contract the script leans on. + const docs = pl.seen.filter((s) => s.path.startsWith("/api/documents/?")); + expect(docs[0].path).toContain("tags__id__all=7"); + expect(docs[0].accept).toBe("application/json; version=10"); + expect(docs[0].auth).toBe("Token pl_test"); + // `next` was followed on the source's own origin. + expect(docs[1].path).toContain("page=2"); + expect( + pl.seen.filter((s) => s.path.includes("/download/")).map((s) => s.path), + ).toEqual([ + "/api/documents/1/download/?original=true", + "/api/documents/2/download/?original=true", + ]); + + // A second run changes nothing: HealthLog answers duplicate for both. + const second = await runScript(args, env); + expect(second.code).toBe(0); + expect(second.stdout).toMatch(/Imported: 0/); + expect(second.stdout).toMatch(/Already in HealthLog: 2/); + }); + + it("leaves a document deleted in HealthLog alone", async () => { + const hl = await healthlog({ deleted: ["PAPERLESS:2"] }); + const pl = await paperless(); + const res = await runScript( + [ + "paperless", + "--paperless-url", + pl.url, + "--healthlog-url", + hl.url, + "--tag", + "healthlog", + ], + { HEALTHLOG_TOKEN: "hlk_test", PAPERLESS_TOKEN: "pl_test" }, + ); + expect(res.code).toBe(0); + expect(res.stdout).toMatch(/Imported: 1/); + expect(res.stdout).toMatch(/Deleted in HealthLog, left alone: 1/); + }); + + it("stops with a plain instruction when the vault is switched off", async () => { + const hl = await healthlog({ moduleOff: true }); + const pl = await paperless(); + const res = await runScript( + ["paperless", "--paperless-url", pl.url, "--healthlog-url", hl.url], + { HEALTHLOG_TOKEN: "hlk_test", PAPERLESS_TOKEN: "pl_test" }, + ); + expect(res.code).toBe(1); + expect(res.stderr).toMatch(/Turn on Documents in Settings/); + }); + + it("lists sizes on a dry run and sends nothing", async () => { + const pl = await paperless(); + const res = await runScript( + [ + "paperless", + "--paperless-url", + pl.url, + "--tag", + "HealthLog", + "--dry-run", + ], + { PAPERLESS_TOKEN: "pl_test" }, + ); + expect(res.code).toBe(0); + expect(res.stdout).toMatch( + /2 document\(s\) would be imported, about 2\.0 MB/, + ); + expect(pl.seen.some((s) => s.path.includes("/download/"))).toBe(false); + }); + + it("refuses to start without a HealthLog token", async () => { + const res = await runScript( + [ + "paperless", + "--paperless-url", + "http://x", + "--healthlog-url", + "http://y", + ], + { PAPERLESS_TOKEN: "pl_test" }, + ); + expect(res.code).toBe(2); + expect(res.stderr).toMatch(/HEALTHLOG_TOKEN is not set/); + }); + + it("names a missing tag instead of importing everything", async () => { + const pl = await paperless(); + const res = await runScript( + ["paperless", "--paperless-url", pl.url, "--tag", "Nope", "--dry-run"], + { PAPERLESS_TOKEN: "pl_test" }, + ); + expect(res.code).toBe(1); + }); +}); + +describe("import-documents.mjs — Papra", () => { + it("imports by tag since a date, and skips what HealthLog refuses as too large", async () => { + const hl = await healthlog({ tooLarge: ["PAPRA:doc_b"] }); + const pp = await papra(); + const res = await runScript( + [ + "papra", + "--papra-url", + pp.url, + "--papra-org", + "org_1", + "--healthlog-url", + hl.url, + "--tag", + "Health", + "--since", + "2016-01-01", + "--kind-map", + "Lab=LAB_RESULT", + "--ai-read", + ], + { HEALTHLOG_TOKEN: "hlk_test", PAPRA_TOKEN: "pp_test" }, + ); + + // Exit 3: finished, with at least one document skipped. + expect(res.code).toBe(3); + const sent = hl.received.map((r) => r.fields); + expect(sent).toHaveLength(2); + expect(sent[0]).toMatchObject({ + title: "Impfpass", + documentDate: "2019-06-01", + sourceSystem: "PAPRA", + sourceId: "doc_a", + }); + // --ai-read: no deferral field at all. + expect(sent[0].aiRead).toBeUndefined(); + expect(sent[1]).toMatchObject({ sourceId: "doc_b", kind: "LAB_RESULT" }); + // doc_old is before --since and never downloaded. + expect(pp.seen.some((s) => s.path.includes("doc_old"))).toBe(false); + expect(pp.seen.find((s) => s.path.includes("/documents?"))?.path).toContain( + "searchQuery=tag%3AHealth", + ); + expect(pp.seen[0].auth).toBe("Bearer pp_test"); + expect(res.stdout).toMatch(/Imported: 1/); + expect(res.stdout).toMatch(/Skipped: 1/); + expect(res.stdout).toMatch(/larger than HealthLog accepts/); + }); +}); diff --git a/scripts/env-manifest.json b/scripts/env-manifest.json index 83b7df229..8a87ecc59 100644 --- a/scripts/env-manifest.json +++ b/scripts/env-manifest.json @@ -138,6 +138,10 @@ "name": "DOCUMENT_AI_LIMIT_PER_HOUR", "purpose": "Per-user hourly ceiling for the document-AI actions (read/index, extract, suggest, summarise share one bucket). Default 6, clamped to 1-1000." }, + { + "name": "DOCUMENT_UPLOAD_LIMIT_PER_HOUR", + "purpose": "Hourly upload ceiling per document-upload token (Paperless workflow, import script). Default 120, clamped to 1-1000. Web and phone uploads keep their own bucket." + }, { "name": "LABS_OCR_LIMIT_PER_HOUR", "purpose": "Per-user hourly ceiling for lab-report scans (photo/PDF and browser-OCR text share one bucket). Default 6, clamped to 1-1000." diff --git a/scripts/import-documents.mjs b/scripts/import-documents.mjs new file mode 100644 index 000000000..305c60b4a --- /dev/null +++ b/scripts/import-documents.mjs @@ -0,0 +1,723 @@ +#!/usr/bin/env node +/** + * Copy documents from Paperless-ngx or Papra into the HealthLog document vault. + * + * Runs wherever you are, not inside the HealthLog container: it talks to two + * HTTP APIs and needs nothing but Node 22 (no dependencies, no install). + * + * node import-documents.mjs paperless --paperless-url https://paperless.lan \ + * --healthlog-url https://health.example --tag HealthLog + * + * node import-documents.mjs papra --papra-url https://papra.lan \ + * --papra-org --healthlog-url https://health.example \ + * --tag Health + * + * Credentials come from the environment, never from flags (flags end up in + * shell history): + * + * HEALTHLOG_TOKEN a document token from HealthLog Settings > API & Tokens + * PAPERLESS_TOKEN a Paperless-ngx API token (My Profile) + * PAPRA_TOKEN a Papra API key with documents:read and tags:read + * + * Options: + * --tag only documents carrying this tag (recommended) + * --since YYYY-MM-DD only documents dated on or after this day + * --kind document type for everything imported + * --kind-map "A=KIND,B=KIND" + * type by Paperless document type or Papra tag name; + * anything unmatched gets --kind, or HealthLog's OTHER + * --dry-run list what would be imported, with sizes; send nothing + * --ai-read let HealthLog read the documents with AI as they + * arrive (by default automatic reading is held back) + * --help this text + * + * One document at a time, one file in memory at a time. A document that is + * already in HealthLog, or that you deleted there, is skipped by HealthLog + * itself, so running the script again is safe and simply picks up where it + * stopped. HealthLog's rate limit answers 429 with Retry-After; the script + * waits and carries on. + */ +import { parseArgs } from "node:util"; +import { setTimeout as sleep } from "node:timers/promises"; + +const KINDS = [ + "DOCTOR_REPORT", + "DISCHARGE_LETTER", + "LAB_RESULT", + "IMAGING", + "PRESCRIPTION", + "REFERRAL", + "INSURANCE", + "VACCINATION", + "OTHER", +]; +const TITLE_MAX = 200; +const SOURCE_ID_MAX = 128; +const PAGE_SIZE = 100; +/** Attempts for a server error or a dropped connection, per request. */ +const TRANSIENT_ATTEMPTS = 3; +const TRANSIENT_BACKOFF_MS = [5_000, 15_000, 45_000]; +/** Longest single wait we accept from a Retry-After header, in seconds. */ +const MAX_RETRY_AFTER_S = 3_600; + +class StopError extends Error {} + +const out = (line = "") => process.stdout.write(`${line}\n`); +const err = (line) => process.stderr.write(`${line}\n`); + +function usage() { + const text = []; + let inside = false; + for (const line of SELF_DOC.split("\n")) { + if (line.startsWith("/**")) inside = true; + else if (line.startsWith(" */")) break; + else if (inside) text.push(line.replace(/^ \* ?/, "")); + } + return text.join("\n"); +} + +// ── Arguments ────────────────────────────────────────────────────────────── + +function readOptions(argv, env) { + const { values, positionals } = parseArgs({ + args: argv, + allowPositionals: true, + options: { + "healthlog-url": { type: "string" }, + "paperless-url": { type: "string" }, + "papra-url": { type: "string" }, + "papra-org": { type: "string" }, + tag: { type: "string" }, + since: { type: "string" }, + kind: { type: "string" }, + "kind-map": { type: "string" }, + "dry-run": { type: "boolean", default: false }, + "ai-read": { type: "boolean", default: false }, + help: { type: "boolean", default: false }, + }, + }); + if (values.help) return { help: true }; + + const source = positionals[0]; + if (source !== "paperless" && source !== "papra") { + throw new StopError( + 'Name the source first: "paperless" or "papra". Run with --help for the options.', + ); + } + const dryRun = values["dry-run"]; + const healthlogUrl = trimSlash(values["healthlog-url"] ?? env.HEALTHLOG_URL); + if (!dryRun && !healthlogUrl) { + throw new StopError("--healthlog-url is required (or HEALTHLOG_URL)."); + } + if (!dryRun && !env.HEALTHLOG_TOKEN) { + throw new StopError( + "HEALTHLOG_TOKEN is not set. Create a document token in HealthLog under Settings > API & Tokens.", + ); + } + if (values.since && !/^\d{4}-\d{2}-\d{2}$/.test(values.since)) { + throw new StopError("--since expects a date as YYYY-MM-DD."); + } + const kind = values.kind ? checkKind(values.kind, "--kind") : null; + const kindMap = new Map(); + if (values["kind-map"]) { + for (const pair of values["kind-map"].split(",")) { + const at = pair.lastIndexOf("="); + if (at <= 0) + throw new StopError(`--kind-map: "${pair}" is not NAME=KIND.`); + kindMap.set( + pair.slice(0, at).trim().toLowerCase(), + checkKind(pair.slice(at + 1).trim(), "--kind-map"), + ); + } + } + + const options = { + source, + healthlogUrl, + healthlogToken: env.HEALTHLOG_TOKEN ?? "", + tag: values.tag ?? null, + since: values.since ?? null, + kind, + kindMap, + dryRun, + aiRead: values["ai-read"], + }; + if (source === "paperless") { + options.baseUrl = trimSlash(values["paperless-url"] ?? env.PAPERLESS_URL); + options.sourceToken = env.PAPERLESS_TOKEN; + if (!options.baseUrl) throw new StopError("--paperless-url is required."); + if (!options.sourceToken) + throw new StopError("PAPERLESS_TOKEN is not set."); + } else { + options.baseUrl = trimSlash(values["papra-url"] ?? env.PAPRA_URL); + options.organizationId = values["papra-org"] ?? env.PAPRA_ORG; + options.sourceToken = env.PAPRA_TOKEN; + if (!options.baseUrl) throw new StopError("--papra-url is required."); + if (!options.organizationId) { + throw new StopError( + "--papra-org is required: the organization id from the Papra address bar.", + ); + } + if (!options.sourceToken) throw new StopError("PAPRA_TOKEN is not set."); + } + return options; +} + +function checkKind(value, flag) { + const upper = value.toUpperCase(); + if (!KINDS.includes(upper)) { + throw new StopError( + `${flag}: "${value}" is not one of ${KINDS.join(", ")}.`, + ); + } + return upper; +} + +function trimSlash(url) { + return url ? url.replace(/\/+$/, "") : url; +} + +// ── HTTP ─────────────────────────────────────────────────────────────────── + +function retryAfterSeconds(response, fallback) { + const raw = response.headers.get("retry-after"); + if (!raw) return fallback; + const seconds = Number(raw); + if (Number.isFinite(seconds)) + return Math.min(Math.max(0, seconds), MAX_RETRY_AFTER_S); + const at = Date.parse(raw); + if (Number.isFinite(at)) { + return Math.min( + Math.max(0, Math.ceil((at - Date.now()) / 1000)), + MAX_RETRY_AFTER_S, + ); + } + return fallback; +} + +/** + * Fetch with the waiting rules shared by all three services: a 429 waits for + * Retry-After and tries again (as often as it takes, since that is the + * service pacing us), a 5xx or a dropped connection is retried a few times + * with a growing pause, and anything else is returned to the caller. + */ +async function request(label, url, init, makeBody) { + let transient = 0; + for (;;) { + let response; + try { + response = await fetch(url, { + ...init, + body: makeBody ? makeBody() : undefined, + redirect: "manual", + }); + } catch (cause) { + if (transient >= TRANSIENT_ATTEMPTS) { + throw new Error(`${label}: could not connect (${cause.message})`); + } + await wait(TRANSIENT_BACKOFF_MS[transient++], `${label} did not answer`); + continue; + } + if (response.status === 429) { + await response.arrayBuffer().catch(() => {}); + const seconds = retryAfterSeconds(response, 60); + await wait(seconds * 1000, `${label} asked to slow down`); + continue; + } + if (response.status >= 500 || response.status === 408) { + await response.arrayBuffer().catch(() => {}); + if (transient >= TRANSIENT_ATTEMPTS) return response; + await wait( + TRANSIENT_BACKOFF_MS[transient++], + `${label} answered ${response.status}`, + ); + continue; + } + return response; + } +} + +async function wait(ms, why) { + if (ms > 0) out(` … ${why}, waiting ${Math.round(ms / 1000)} s`); + await sleep(ms); +} + +async function sourceJson(label, url, headers) { + const response = await request(label, url, { headers }); + if (response.status === 401 || response.status === 403) { + throw new StopError( + `${label} refused the token (${response.status}). Check the token and its permissions.`, + ); + } + if (!response.ok) { + throw new StopError(`${label} answered ${response.status} for ${url}`); + } + return response.json(); +} + +/** Keep the source's own origin: a `next` link behind a proxy may name another. */ +function sameOrigin(base, link) { + const next = new URL(link, base); + const origin = new URL(base); + next.protocol = origin.protocol; + next.host = origin.host; + return next.toString(); +} + +function filenameFrom(response, fallback) { + const header = response.headers.get("content-disposition") ?? ""; + const star = /filename\*=(?:UTF-8'')?([^;]+)/i.exec(header); + if (star) { + try { + return decodeURIComponent(star[1].trim().replace(/^"|"$/g, "")); + } catch { + /* fall through */ + } + } + const plain = /filename="?([^";]+)"?/i.exec(header); + return plain ? plain[1] : fallback; +} + +// ── Paperless-ngx ────────────────────────────────────────────────────────── + +function paperless(options) { + const headers = { + Authorization: `Token ${options.sourceToken}`, + Accept: "application/json; version=10", + }; + const base = options.baseUrl; + const label = "Paperless-ngx"; + + async function* pages(firstUrl) { + let url = firstUrl; + while (url) { + const page = await sourceJson(label, url, headers); + for (const item of page.results ?? []) yield item; + url = page.next ? sameOrigin(base, page.next) : null; + } + } + + return { + system: "PAPERLESS", + label, + async *documents() { + const query = new URLSearchParams({ + page_size: String(PAGE_SIZE), + ordering: "id", + }); + let tagId = null; + if (options.tag) { + const found = []; + for await (const tag of pages( + `${base}/api/tags/?${new URLSearchParams({ name__iexact: options.tag, page_size: String(PAGE_SIZE) })}`, + )) { + if (String(tag.name).toLowerCase() === options.tag.toLowerCase()) + found.push(tag); + } + if (found.length === 0) { + throw new StopError( + `Paperless-ngx has no tag named "${options.tag}".`, + ); + } + tagId = found[0].id; + query.set("tags__id__all", String(tagId)); + } + if (options.since) query.set("created__date__gte", options.since); + + let typeNames = new Map(); + if (options.kindMap.size > 0) { + for await (const type of pages( + `${base}/api/document_types/?page_size=${PAGE_SIZE}`, + )) { + typeNames.set(type.id, String(type.name)); + } + } + + for await (const doc of pages(`${base}/api/documents/?${query}`)) { + // The server filters too; checking again keeps an older Paperless + // that ignores a parameter from importing the whole archive. + if ( + tagId !== null && + Array.isArray(doc.tags) && + !doc.tags.includes(tagId) + ) { + continue; + } + const created = dayOf(doc.created); + const added = dayOf(doc.added); + yield { + sourceId: String(doc.id), + title: doc.title || doc.original_file_name || `Paperless ${doc.id}`, + date: created ?? added, + dateFallback: created ? null : added ? "added" : "none", + filename: doc.original_file_name ?? null, + kindKeys: + doc.document_type != null && typeNames.has(doc.document_type) + ? [typeNames.get(doc.document_type)] + : [], + size: null, + }; + } + }, + async size(doc) { + const meta = await sourceJson( + label, + `${base}/api/documents/${encodeURIComponent(doc.sourceId)}/metadata/`, + headers, + ); + return typeof meta.original_size === "number" ? meta.original_size : null; + }, + async download(doc) { + const response = await request( + label, + `${base}/api/documents/${encodeURIComponent(doc.sourceId)}/download/?original=true`, + { headers: { Authorization: headers.Authorization } }, + ); + return readDownload(label, response, doc); + }, + }; +} + +// ── Papra ────────────────────────────────────────────────────────────────── + +function papra(options) { + const headers = { + Authorization: `Bearer ${options.sourceToken}`, + Accept: "application/json", + }; + const org = `${options.baseUrl}/api/organizations/${encodeURIComponent(options.organizationId)}`; + const label = "Papra"; + + return { + system: "PAPRA", + label, + async *documents() { + let tagId = null; + if (options.tag) { + const body = await sourceJson(label, `${org}/tags`, headers); + const tag = (body.tags ?? []).find( + (t) => String(t.name).toLowerCase() === options.tag.toLowerCase(), + ); + if (!tag) + throw new StopError(`Papra has no tag named "${options.tag}".`); + tagId = tag.id; + } + const search = options.tag + ? `tag:${/\s/.test(options.tag) ? JSON.stringify(options.tag) : options.tag}` + : ""; + for (let pageIndex = 0; ; pageIndex++) { + const query = new URLSearchParams({ + pageIndex: String(pageIndex), + pageSize: String(PAGE_SIZE), + }); + if (search) query.set("searchQuery", search); + const body = await sourceJson( + label, + `${org}/documents?${query}`, + headers, + ); + const documents = body.documents ?? []; + for (const doc of documents) { + const tags = Array.isArray(doc.tags) ? doc.tags : null; + if (tagId !== null && tags && !tags.some((t) => t.id === tagId)) + continue; + const documentDate = dayOf(doc.documentDate); + const created = dayOf(doc.createdAt); + const date = documentDate ?? created; + // Papra's search dates differ between versions; the date is + // checked here instead of trusting a query syntax. + if (options.since && date && date < options.since) continue; + yield { + sourceId: String(doc.id), + title: stripExtension(doc.name) || `Papra ${doc.id}`, + date, + dateFallback: documentDate ? null : created ? "added" : "none", + filename: doc.originalName ?? doc.name ?? null, + kindKeys: (tags ?? []).map((t) => String(t.name)), + size: + typeof doc.originalSize === "number" ? doc.originalSize : null, + }; + } + const total = + typeof body.documentsCount === "number" ? body.documentsCount : null; + if (documents.length < PAGE_SIZE) break; + if (total !== null && (pageIndex + 1) * PAGE_SIZE >= total) break; + } + }, + async size(doc) { + return doc.size; + }, + async download(doc) { + const response = await request( + label, + `${org}/documents/${encodeURIComponent(doc.sourceId)}/file`, + { headers: { Authorization: headers.Authorization } }, + ); + return readDownload(label, response, doc); + }, + }; +} + +async function readDownload(label, response, doc) { + if (response.status === 401 || response.status === 403) { + throw new StopError(`${label} refused the download (${response.status}).`); + } + if (!response.ok) { + return { skip: `${label} could not deliver the file (${response.status})` }; + } + const bytes = new Uint8Array(await response.arrayBuffer()); + return { + bytes, + filename: + doc.filename ?? filenameFrom(response, `document-${doc.sourceId}`), + type: response.headers.get("content-type") ?? "application/octet-stream", + }; +} + +function dayOf(value) { + if (typeof value !== "string") return null; + const day = value.slice(0, 10); + return /^\d{4}-\d{2}-\d{2}$/.test(day) ? day : null; +} + +function stripExtension(name) { + if (typeof name !== "string") return ""; + return name.replace(/\.[A-Za-z0-9]{1,5}$/, ""); +} + +// ── HealthLog ────────────────────────────────────────────────────────────── + +function kindFor(options, doc) { + for (const key of doc.kindKeys) { + const mapped = options.kindMap.get(key.toLowerCase()); + if (mapped) return mapped; + } + return options.kind; +} + +async function upload(options, source, doc, file) { + const url = `${options.healthlogUrl}/api/documents/inbound`; + const kind = kindFor(options, doc); + const response = await request( + "HealthLog", + url, + { + method: "POST", + headers: { Authorization: `Bearer ${options.healthlogToken}` }, + }, + // Rebuilt per attempt: a body stream is spent once it has been sent. + () => { + const form = new FormData(); + form.append( + "file", + new Blob([file.bytes], { type: file.type }), + file.filename, + ); + form.append("title", doc.title.trim().slice(0, TITLE_MAX)); + if (doc.date) form.append("documentDate", doc.date); + if (kind) form.append("kind", kind); + form.append("sourceSystem", source.system); + form.append("sourceId", doc.sourceId.slice(0, SOURCE_ID_MAX)); + if (!options.aiRead) form.append("aiRead", "defer"); + return form; + }, + ); + + let body = null; + try { + body = await response.json(); + } catch { + body = null; + } + const meta = body?.meta ?? {}; + switch (response.status) { + case 201: + return { outcome: "imported" }; + case 200: + return body?.data?.deleted + ? { outcome: "deleted" } + : { outcome: "duplicate" }; + case 401: + throw new StopError( + "HealthLog refused the token (401). It may be expired or revoked; create a new document token.", + ); + case 403: + if (meta.errorCode === "module.disabled") { + throw new StopError( + "The document vault is switched off in HealthLog. Turn on Documents in Settings, then run the script again.", + ); + } + throw new StopError( + "HealthLog refused the upload (403). Use a document token from Settings > API & Tokens; other tokens cannot upload documents.", + ); + case 413: + if (meta.reason === "quotaExceeded") { + throw new StopError( + `Your HealthLog storage is full (${mib(meta.usedBytes)} of ${mib(meta.quotaBytes)} used). An admin can raise the limit in the admin area, for everyone or just for your account.`, + ); + } + return { + outcome: "skipped", + reason: `larger than HealthLog accepts${meta.maxFileBytes ? ` (${mib(meta.maxFileBytes)})` : ""}`, + }; + case 415: + return { + outcome: "skipped", + reason: "HealthLog does not accept this file type", + }; + case 422: + return { + outcome: "skipped", + reason: `HealthLog refused the details: ${body?.error ?? "invalid"}`, + }; + default: + return { + outcome: "skipped", + reason: `HealthLog answered ${response.status}${body?.error ? `: ${body.error}` : ""}`, + }; + } +} + +function mib(bytes) { + if (typeof bytes !== "number") return "?"; + return bytes >= 1024 * 1024 * 1024 + ? `${(bytes / 1024 ** 3).toFixed(1)} GB` + : `${(bytes / 1024 ** 2).toFixed(1)} MB`; +} + +// ── Run ──────────────────────────────────────────────────────────────────── + +async function run(argv, env = process.env) { + let options; + try { + options = readOptions(argv, env); + } catch (error) { + if ( + error instanceof StopError || + error?.code === "ERR_PARSE_ARGS_UNKNOWN_OPTION" + ) { + err(error.message); + return 2; + } + throw error; + } + if (options.help) { + out(usage()); + return 0; + } + + const source = + options.source === "paperless" ? paperless(options) : papra(options); + const tally = { + imported: 0, + duplicate: 0, + deleted: 0, + skipped: [], + fallback: [], + }; + let listed = 0; + let totalBytes = 0; + let unknownSizes = 0; + + out( + `${options.dryRun ? "Dry run: listing" : "Importing"} documents from ${source.label}` + + `${options.tag ? ` tagged "${options.tag}"` : ""}${options.since ? ` since ${options.since}` : ""}.`, + ); + + try { + for await (const doc of source.documents()) { + listed++; + const where = `${source.label} ${doc.sourceId} "${doc.title}"`; + if (doc.dateFallback) + tally.fallback.push({ where, dateFallback: doc.dateFallback }); + + if (options.dryRun) { + const size = await source.size(doc); + if (size === null) unknownSizes++; + else totalBytes += size; + const kind = kindFor(options, doc) ?? "OTHER"; + out( + ` ${doc.date ?? "no date"} ${kind.padEnd(16)} ${size === null ? "?" : mib(size)} ${where}`, + ); + continue; + } + + const file = await source.download(doc); + if (file.skip) { + tally.skipped.push({ where, reason: file.skip }); + out(` skipped ${where}: ${file.skip}`); + continue; + } + const result = await upload(options, source, doc, file); + if (result.outcome === "imported") { + tally.imported++; + out(` imported ${where}`); + } else if (result.outcome === "duplicate") { + tally.duplicate++; + out(` present ${where}`); + } else if (result.outcome === "deleted") { + tally.deleted++; + out(` deleted ${where} (you deleted it in HealthLog; left alone)`); + } else { + tally.skipped.push({ where, reason: result.reason }); + out(` skipped ${where}: ${result.reason}`); + } + } + } catch (error) { + printSummary(options, tally, listed, totalBytes, unknownSizes); + err(""); + err(`Stopped: ${error.message}`); + return 1; + } + + printSummary(options, tally, listed, totalBytes, unknownSizes); + return tally.skipped.length > 0 ? 3 : 0; +} + +function printSummary(options, tally, listed, totalBytes, unknownSizes) { + out(""); + if (options.dryRun) { + out( + `${listed} document(s) would be imported, about ${mib(totalBytes)}` + + `${unknownSizes > 0 ? ` (size unknown for ${unknownSizes})` : ""}.`, + ); + out( + "HealthLog's default storage limit is 1 GB per person; an admin can raise it.", + ); + out( + "Documents already in HealthLog, or deleted there, will be skipped by HealthLog.", + ); + } else { + out(`Imported: ${tally.imported}`); + out(`Already in HealthLog: ${tally.duplicate}`); + out(`Deleted in HealthLog, left alone: ${tally.deleted}`); + out(`Skipped: ${tally.skipped.length}`); + for (const { where, reason } of tally.skipped) out(` ${where}: ${reason}`); + } + if (tally.fallback.length > 0) { + out( + `No document date in the source for ${tally.fallback.length} document(s):`, + ); + for (const { where, dateFallback } of tally.fallback) { + out( + dateFallback === "added" + ? ` ${where}: filed under the day it was added to the source` + : ` ${where}: filed under the day of import`, + ); + } + } + if (!options.dryRun && tally.imported > 0 && !options.aiRead) { + out( + `${tally.imported} document(s) imported without automatic AI reading. ` + + "To have one read, open it in HealthLog and choose Read with AI or Generate summary; " + + "Documents > Index all for search reads the ones that had no text to search.", + ); + } +} + +const SELF_DOC = await (async () => { + const { readFile } = await import("node:fs/promises"); + return readFile(new URL(import.meta.url), "utf8"); +})(); + +process.exitCode = await run(process.argv.slice(2)); diff --git a/src/__tests__/bearer-scope-enforcement-guard.test.ts b/src/__tests__/bearer-scope-enforcement-guard.test.ts index dcc05ca09..2e28046f5 100644 --- a/src/__tests__/bearer-scope-enforcement-guard.test.ts +++ b/src/__tests__/bearer-scope-enforcement-guard.test.ts @@ -149,6 +149,9 @@ describe("T3 — the mint sites are frozen", () => { // body cannot express a scope. Reaches two write routes on the holder's // own record and nothing else, the mint included. "app/api/tokens/measurements/route.ts": "[MEASUREMENTS_WRITE_SCOPE]", + // Document upload from another system (#1038). Same shape as the + // measurement mint: a literal, one scope, one route that accepts it. + "app/api/tokens/documents/route.ts": "[DOCUMENTS_WRITE_SCOPE]", // MCP, audience-bound to `/mcp`. `health:write` requires explicit consent. "app/api/mcp/tokens/route.ts": "SCOPE_HEALTH_READ / SCOPE_HEALTH_WRITE", "app/api/mcp/oauth/token/route.ts": @@ -175,6 +178,7 @@ describe("T3 — the mint sites are frozen", () => { const userFacing = [ "app/api/medications/[id]/api-endpoint/route.ts", "app/api/tokens/measurements/route.ts", + "app/api/tokens/documents/route.ts", "app/api/mcp/tokens/route.ts", "app/api/mcp/oauth/token/route.ts", ]; @@ -226,7 +230,11 @@ describe("T4 — declared scopes are exported constants, never string literals", // `scope` option, a route can widen itself without the word `requireAuth` // appearing anywhere in it. A vocabulary that watched only the first form // would have gone on passing while the surface it describes grew. - const DECLARED_SCOPES = ["FHIR_READ_SCOPE", "MEASUREMENTS_WRITE_SCOPE"]; + const DECLARED_SCOPES = [ + "DOCUMENTS_WRITE_SCOPE", + "FHIR_READ_SCOPE", + "MEASUREMENTS_WRITE_SCOPE", + ]; const used = new Set(); for (const rel of sourceFiles()) { const src = read(rel); @@ -245,6 +253,24 @@ describe("T4 — declared scopes are exported constants, never string literals", }); }); +describe("T4b — `documents:write` opens exactly one door", () => { + it("only the vault upload route declares the scope", () => { + // The scope admits a narrow token wherever it is named, so the set of + // files naming it IS its reach. The upload, and nothing else: not the + // list or detail beside it, not the original or thumbnail, not bulk, not + // the AI legs. A second route here is a widening and belongs in review. + const declaring = filesMatching( + /requireAuth\(\s*DOCUMENTS_WRITE_SCOPE\s*\)|requireRecordAuth\([^)]*\bscope:\s*DOCUMENTS_WRITE_SCOPE\b/, + ); + expect(declaring).toEqual(["app/api/documents/inbound/route.ts"]); + // And within that file, once: the GET beside the POST must not name it. + const src = read("app/api/documents/inbound/route.ts"); + expect( + src.match(/requireAuth\(\s*DOCUMENTS_WRITE_SCOPE\s*\)/g), + ).toHaveLength(1); + }); +}); + describe("T5 — a delegable route naming a Bearer scope is frozen", () => { /** * Routes that admit a narrow token on a DELEGABLE surface, and the scope each diff --git a/src/app/api/documents/inbound/__tests__/route.test.ts b/src/app/api/documents/inbound/__tests__/route.test.ts index 265bc6e6d..b144b1420 100644 --- a/src/app/api/documents/inbound/__tests__/route.test.ts +++ b/src/app/api/documents/inbound/__tests__/route.test.ts @@ -1,4 +1,4 @@ -import { beforeEach, describe, expect, it, vi } from "vitest"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; /** * Document vault: store-only upload + browsable list. @@ -50,6 +50,9 @@ vi.mock("@/lib/db", () => { findMany: vi.fn(), findFirst: vi.fn(), }, + documentImportKey: { + findUnique: vi.fn(), + }, extractedFact: { groupBy: vi.fn(), }, @@ -114,6 +117,12 @@ vi.mock("@/lib/rate-limit", () => ({ })); vi.mock("@/lib/auth/session", () => ({ getSession: vi.fn() })); +// The Bearer path: the real `requireAuth` runs, and only the token lookup is +// stubbed, so the scope decision under test is the production one. +vi.mock("@/lib/auth/bearer", async (importOriginal) => { + const actual = await importOriginal(); + return { ...actual, resolveBearerToken: vi.fn() }; +}); vi.mock("@/lib/auth/audit", () => ({ auditLog: vi.fn().mockResolvedValue(undefined), })); @@ -135,6 +144,11 @@ import { createHash } from "node:crypto"; import { POST, GET } from "../route"; import { prisma } from "@/lib/db"; import { getSession } from "@/lib/auth/session"; +import { headers } from "next/headers"; +import { resolveBearerToken, BearerAuthError } from "@/lib/auth/bearer"; +import { checkRateLimit } from "@/lib/rate-limit"; +import { enqueueDocumentThumbnail } from "@/lib/jobs/document-thumbnail"; +import { DOCUMENTS_WRITE_SCOPE } from "@/lib/documents/scopes"; import { requireModuleEnabled } from "@/lib/modules/gate"; import { enqueueDocumentIndex } from "@/lib/jobs/document-index"; import { enqueueDocumentSummary } from "@/lib/jobs/document-summary"; @@ -219,6 +233,9 @@ beforeEach(() => { [] as never, ); vi.mocked(prisma.inboundDocument.findFirst).mockResolvedValue(null as never); + vi.mocked(prisma.documentImportKey.findUnique).mockResolvedValue( + null as never, + ); // No settings row / no user override → policy defaults (25 MiB / 1 GiB). vi.mocked(prisma.appSettings.findUnique).mockResolvedValue(null as never); vi.mocked(prisma.user.findUnique).mockResolvedValue(null as never); @@ -581,3 +598,216 @@ describe("POST /api/documents/inbound — AI work on upload follows documentAi", expect(enqueueDocumentSummary).not.toHaveBeenCalled(); }); }); + +/** + * v1.39.2 (#1038) — the narrow `documents:write` token, and the import fields + * any caller may send. The Bearer is resolved by the real `requireAuth`; only + * the token lookup is stubbed. + */ +describe("POST /api/documents/inbound — document import", () => { + const TOKEN = "hlk_" + "d".repeat(64); + + function armToken(permissions: string[]) { + vi.mocked(getSession).mockResolvedValue(null as never); + vi.mocked(headers).mockResolvedValue({ + get: (name: string) => + name.toLowerCase() === "authorization" ? `Bearer ${TOKEN}` : null, + } as never); + vi.mocked(resolveBearerToken).mockImplementation(async (_raw, req) => { + const admitted = + permissions.includes("*") || + (req.kind === "scope" && permissions.includes(req.scope)); + if (!admitted) { + throw new BearerAuthError( + 403, + "insufficient_permissions", + "user-1", + "tok-1", + ); + } + return { + user: { id: "user-1", username: "tester", role: "USER" }, + tokenId: "tok-1", + expiresAt: new Date(Date.now() + 3_600_000), + permissions, + } as never; + }); + } + + beforeEach(() => { + vi.mocked(checkRateLimit).mockResolvedValue({ + allowed: true, + remaining: 59, + resetAt: Date.now() + 3_600_000, + } as never); + }); + + afterEach(() => { + vi.mocked(headers).mockResolvedValue({ get: () => null } as never); + }); + + it("admits the scoped token and answers with a receipt, not the row", async () => { + armToken([DOCUMENTS_WRITE_SCOPE]); + txCreate.mockResolvedValue( + docRow({ title: "Befund", filename: "befund.png" }) as never, + ); + + const res = await post(mkUpload({ title: "Befund" })); + expect(res.status).toBe(201); + const body = await res.json(); + // Exactly the receipt: no title, filename, kind, links or counts. + expect(body.data).toEqual({ id: "doc-1", duplicate: false }); + }); + + it("gives the scoped token its own bucket, keyed on the token", async () => { + armToken([DOCUMENTS_WRITE_SCOPE]); + await post(mkUpload()); + const key = vi.mocked(checkRateLimit).mock.calls[0]![0]; + expect(key).toBe("documents-upload:token:tok-1"); + // Default 120/hour for a token; the cookie bucket stays at 60. + expect(vi.mocked(checkRateLimit).mock.calls[0]![1]).toBe(120); + }); + + it("keeps the cookie bucket for a session", async () => { + await post(mkUpload()); + expect(vi.mocked(checkRateLimit).mock.calls[0]![0]).toBe( + "documents-upload:user-1", + ); + expect(vi.mocked(checkRateLimit).mock.calls[0]![1]).toBe(60); + }); + + it("429s the scoped token past its bucket with Retry-After", async () => { + armToken([DOCUMENTS_WRITE_SCOPE]); + vi.mocked(checkRateLimit).mockResolvedValue({ + allowed: false, + remaining: 0, + limit: 120, + resetAt: Date.now() + 90_000, + } as never); + const res = await post(mkUpload()); + expect(res.status).toBe(429); + expect((await res.json()).meta.errorCode).toBe( + "documents.inbound.rateLimited", + ); + expect(txCreate).not.toHaveBeenCalled(); + }); + + it("refuses a token carrying some other narrow scope", async () => { + armToken(["measurements:write"]); + const res = await post(mkUpload()); + expect(res.status).toBe(403); + expect(txCreate).not.toHaveBeenCalled(); + }); + + it("answers a scoped content duplicate with the receipt", async () => { + armToken([DOCUMENTS_WRITE_SCOPE]); + vi.mocked(prisma.inboundDocument.findFirst).mockResolvedValue( + docRow({ id: "doc-existing", title: "Private title" }) as never, + ); + const res = await post(mkUpload()); + expect(res.status).toBe(200); + const body = await res.json(); + expect(body.data).toEqual({ id: "doc-existing", duplicate: true }); + expect(body.meta.duplicate).toBe(true); + }); + + it("stores the source key on a fresh import", async () => { + armToken([DOCUMENTS_WRITE_SCOPE]); + await post(mkUpload({ sourceSystem: "PAPERLESS", sourceId: "412" })); + const arg = txCreate.mock.calls[0]![0]!; + expect(arg.data.sourceSystem).toBe("PAPERLESS"); + expect(arg.data.sourceId).toBe("412"); + }); + + it("answers a live source-key match as a duplicate without reading bytes", async () => { + vi.mocked(prisma.inboundDocument.findFirst).mockResolvedValueOnce( + docRow({ + id: "doc-src", + sourceSystem: "PAPRA", + sourceId: "doc_1", + }) as never, + ); + const res = await post( + mkUpload({ sourceSystem: "PAPRA", sourceId: "doc_1" }), + ); + expect(res.status).toBe(200); + const body = await res.json(); + expect(body.meta.duplicate).toBe(true); + expect(body.data.id).toBe("doc-src"); + // The lookup spans tombstones: no deletedAt predicate. + const where = vi.mocked(prisma.inboundDocument.findFirst).mock.calls[0]![0]! + .where!; + expect(where).toEqual({ + userId: "user-1", + sourceSystem: "PAPRA", + sourceId: "doc_1", + }); + expect(txCreate).not.toHaveBeenCalled(); + }); + + it("refuses to bring back a deleted import: 200 deleted, no new row", async () => { + vi.mocked(prisma.inboundDocument.findFirst).mockResolvedValueOnce( + docRow({ + id: "doc-gone", + sourceSystem: "PAPERLESS", + sourceId: "7", + deletedAt: new Date("2026-09-01T00:00:00.000Z"), + }) as never, + ); + const res = await post( + mkUpload({ sourceSystem: "PAPERLESS", sourceId: "7" }), + ); + expect(res.status).toBe(200); + const body = await res.json(); + expect(body.data).toEqual({ + id: "doc-gone", + duplicate: true, + deleted: true, + }); + expect(body.meta).toEqual({ duplicate: true, deleted: true }); + expect(txCreate).not.toHaveBeenCalled(); + expect(enqueueDocumentIndex).not.toHaveBeenCalled(); + }); + + it("remembers a purged import through the ledger", async () => { + vi.mocked(prisma.documentImportKey.findUnique).mockResolvedValue({ + id: "key-1", + } as never); + const res = await post( + mkUpload({ sourceSystem: "PAPERLESS", sourceId: "7" }), + ); + expect(res.status).toBe(200); + const body = await res.json(); + expect(body.data).toEqual({ id: null, duplicate: true, deleted: true }); + expect(txCreate).not.toHaveBeenCalled(); + }); + + it("422s a sourceId without a sourceSystem", async () => { + const res = await post(mkUpload({ sourceId: "7" })); + expect(res.status).toBe(422); + expect(txCreate).not.toHaveBeenCalled(); + }); + + it("422s a sourceSystem outside the closed set", async () => { + const res = await post( + mkUpload({ sourceSystem: "DROPBOX", sourceId: "7" }), + ); + expect(res.status).toBe(422); + }); + + it("aiRead=defer indexes locally and skips the summary", async () => { + const res = await post(mkUpload({ aiRead: "defer" })); + expect(res.status).toBe(201); + expect(enqueueDocumentIndex).toHaveBeenCalledWith("user-1", "doc-1", { + localOnly: true, + }); + expect(enqueueDocumentThumbnail).toHaveBeenCalledWith("user-1", "doc-1"); + expect(enqueueDocumentSummary).not.toHaveBeenCalled(); + }); + + it("without aiRead the upload keeps today's AI behaviour", async () => { + await post(mkUpload()); + expect(enqueueDocumentIndex).toHaveBeenCalledWith("user-1", "doc-1"); + expect(enqueueDocumentSummary).toHaveBeenCalledWith("user-1", "doc-1"); + }); +}); diff --git a/src/app/api/documents/inbound/route.ts b/src/app/api/documents/inbound/route.ts index a10dcc3ba..e38c5d284 100644 --- a/src/app/api/documents/inbound/route.ts +++ b/src/app/api/documents/inbound/route.ts @@ -15,6 +15,15 @@ * (a same-user re-upload returns the existing live row, never a second copy), * an honoured `Idempotency-Key`, and optional `episodeIds[]` pre-linking. * + * v1.39.2 (#1038) — POST also admits a narrow `documents:write` token, the + * door another document system (a Paperless workflow, the import script) + * pushes through. Such a caller gets its own rate bucket, and a receipt + * (`{ id, duplicate }`) instead of the stored row: a credential that can only + * add files learns that a file exists, never how its owner filed it. Optional + * `sourceSystem` / `sourceId` fields key an import so a re-send is a + * duplicate, including after the person deleted the document; `aiRead=defer` + * holds back automatic AI reading for the upload. + * * GET lists the caller's documents with title/filename search, kind / * episode / year / date-range filters, sort, and keyset pagination — and * NEVER selects the encrypted blob column (`omit: { contentEncrypted: true }`). @@ -29,6 +38,7 @@ import { NextResponse } from "next/server"; import { apiHandler, + isScopedCredential, requireAuth, requireRecordAuth, type AuthContext, @@ -58,10 +68,12 @@ import { import { enqueueDocumentIndex } from "@/lib/jobs/document-index"; import { enqueueDocumentSummary } from "@/lib/jobs/document-summary"; import { enqueueDocumentThumbnail } from "@/lib/jobs/document-thumbnail"; +import { DOCUMENTS_WRITE_SCOPE } from "@/lib/documents/scopes"; import { acquireDocumentUploadSlot, detectDocumentType, resolveDocumentLimits, + resolveDocumentUploadLimitPerHour, } from "@/lib/documents/upload-policy"; import { withIdempotency } from "@/lib/idempotency"; import { BodyTooLargeError, readBoundedBody } from "@/lib/labs/ocr-upload"; @@ -74,6 +86,7 @@ import { documentCreateSchema, documentListQuerySchema, toContentIndexSource, + type DocumentSourceSystemValue, } from "@/lib/validations/inbound-documents"; import type { Prisma } from "@/generated/prisma/client"; @@ -116,15 +129,93 @@ class QuotaExceededError extends Error { } } +/** What a single upload asked for, carried into the annotations. */ +interface UploadContext { + userId: string; + /** A narrow `documents:write` token, not a session or a `["*"]` token. */ + scoped: boolean; + sourceSystem: DocumentSourceSystemValue | null; + aiDeferred: boolean; +} + +/** + * The receipt a scoped caller gets in place of the stored row: the id and + * whether the upload was a duplicate. A write-only credential holder who + * knows a file's bytes learns that it is stored, not its title, filename, + * links or fact counts. + */ +function receiptResponse( + id: string | null, + status: 200 | 201, + flags: { duplicate: boolean; deleted?: boolean }, +): NextResponse { + const data = { + id, + duplicate: flags.duplicate, + ...(flags.deleted ? { deleted: true } : {}), + }; + return NextResponse.json( + { + data, + error: null, + ...(flags.duplicate + ? { + meta: { + duplicate: true, + ...(flags.deleted ? { deleted: true } : {}), + }, + } + : {}), + }, + { status }, + ); +} + +/** + * v1.39.2 (#1038) — a re-send of an imported document the person deleted. + * 200 and no new row, for every caller: the deletion was a decision, and an + * importer re-running over the same source must not undo it. `id` is the + * tombstoned row while it exists and null once the purge took it; there is no + * row left to serialise either way, so the answer is the receipt. + */ +function deletedResponse(ctx: UploadContext, id: string | null): NextResponse { + annotate({ + action: { name: "documents.vault.upload" }, + meta: { + documentId: id, + duplicate: true, + deleted: true, + scoped: ctx.scoped, + sourceSystem: ctx.sourceSystem, + aiDeferred: ctx.aiDeferred, + }, + }); + return receiptResponse(id, 200, { duplicate: true, deleted: true }); +} + /** * §3.2 — a duplicate upload is NOT an error: return the existing live row * with `meta.duplicate: true` at the envelope level (the UI toasts "already - * stored" and highlights the row). + * stored" and highlights the row). A scoped caller gets the receipt instead. */ async function duplicateResponse( - userId: string, + ctx: UploadContext, existing: SerialisableDocument, ): Promise { + annotate({ + action: { name: "documents.vault.upload" }, + meta: { + documentId: existing.id, + duplicate: true, + scoped: ctx.scoped, + sourceSystem: ctx.sourceSystem, + aiDeferred: ctx.aiDeferred, + }, + }); + if (ctx.scoped) { + return receiptResponse(existing.id, 200, { duplicate: true }); + } + const userId = ctx.userId; const [links, visitLinks, groups] = await Promise.all([ loadConditionLinks(userId, [existing.id]), loadDocumentEncounterLinks(userId, [existing.id]), @@ -140,10 +231,6 @@ async function duplicateResponse( if (g.status !== "REJECTED") factCount += g._count._all; if (g.status === "PENDING") pendingCount += g._count._all; } - annotate({ - action: { name: "documents.vault.upload" }, - meta: { documentId: existing.id, duplicate: true }, - }); return NextResponse.json( { data: serialiseDocument( @@ -165,18 +252,31 @@ async function duplicateResponse( /** Process one admitted upload. The caller owns and releases its memory slot. */ async function processUpload( request: Request, - user: AuthContext["user"], + auth: AuthContext, ): Promise { + const { user } = auth; + const scoped = isScopedCredential(auth); + // Opt-in module gate — even a valid Bearer token is refused when the surface // is off (it ships dark; the user turns it on deliberately). const gate = await requireModuleEnabled(user.id, "inboundDocuments"); if (!gate.enabled) return gate.response; - const rl = await checkRateLimit( - `documents-upload:${user.id}`, - UPLOAD_LIMIT_PER_HOUR, - UPLOAD_WINDOW_MS, - ); + // A scoped token draws on its own bucket, keyed on the token (its id rides + // `session.id` on the Bearer path), so an import running overnight cannot + // use up the allowance the person's own uploads from the web or the phone + // draw on. The cookie / wildcard bucket is unchanged. + const rl = scoped + ? await checkRateLimit( + `documents-upload:token:${auth.session.id}`, + resolveDocumentUploadLimitPerHour(), + UPLOAD_WINDOW_MS, + ) + : await checkRateLimit( + `documents-upload:${user.id}`, + UPLOAD_LIMIT_PER_HOUR, + UPLOAD_WINDOW_MS, + ); if (!rl.allowed) { const response = apiError("Too many uploads. Try again later.", 429, { errorCode: "documents.inbound.rateLimited", @@ -241,6 +341,9 @@ async function processUpload( documentDate: formData.get("documentDate") ?? undefined, episodeIds: rawEpisodeIds.length > 0 ? rawEpisodeIds : undefined, encounterIds: rawEncounterIds.length > 0 ? rawEncounterIds : undefined, + sourceSystem: formData.get("sourceSystem") ?? undefined, + sourceId: formData.get("sourceId") ?? undefined, + aiRead: formData.get("aiRead") ?? undefined, }); if (!parsed.success) { return apiValidationError( @@ -253,6 +356,39 @@ async function processUpload( ); } + const sourceSystem = parsed.data.sourceSystem ?? null; + const sourceId = sourceSystem ? (parsed.data.sourceId ?? null) : null; + const ctx: UploadContext = { + userId: user.id, + scoped, + sourceSystem, + aiDeferred: parsed.data.aiRead === "defer", + }; + + // A source key answers before the bytes are looked at: an import re-sending + // what it sent before is a duplicate, and one re-sending a document the + // person deleted is refused a second copy — live and tombstoned rows both + // count (the unique index has no `deleted_at` predicate), and past the purge + // the ledger remembers. + if (sourceSystem && sourceId) { + const keyed = await findBySourceKey(user.id, sourceSystem, sourceId); + if (keyed) + return keyed.deletedAt + ? deletedResponse(ctx, keyed.id) + : duplicateResponse(ctx, keyed); + const purged = await prisma.documentImportKey.findUnique({ + where: { + userId_sourceSystem_sourceId: { + userId: user.id, + sourceSystem, + sourceId, + }, + }, + select: { id: true }, + }); + if (purged) return deletedResponse(ctx, null); + } + let buffer: Buffer; try { buffer = Buffer.from(await file.arrayBuffer()); @@ -313,7 +449,7 @@ async function processUpload( omit: { contentEncrypted: true }, }); if (existing) { - return duplicateResponse(user.id, existing); + return duplicateResponse(ctx, existing); } // Quota gate + insert + pre-links in ONE transaction. Usage counts every @@ -369,6 +505,8 @@ async function processUpload( documentDate: parsed.data.documentDate ? isoDateToUtc(parsed.data.documentDate) : new Date(), + sourceSystem, + sourceId, }, omit: { contentEncrypted: true }, }); @@ -403,13 +541,22 @@ async function processUpload( }); } if (isP2002(err)) { - // A racing upload of the same bytes won the partial unique index. - // Surface the winner as the duplicate — same outcome as the fast path. + // A racing upload won one of the two partial unique indexes — the same + // source key, or the same bytes. Surface the winner exactly as the fast + // paths above would have. + if (sourceSystem && sourceId) { + const keyed = await findBySourceKey(user.id, sourceSystem, sourceId); + if (keyed) { + return keyed.deletedAt + ? deletedResponse(ctx, keyed.id) + : duplicateResponse(ctx, keyed); + } + } const winner = await prisma.inboundDocument.findFirst({ where: { userId: user.id, contentSha256, deletedAt: null }, omit: { contentEncrypted: true }, }); - if (winner) return duplicateResponse(user.id, winner); + if (winner) return duplicateResponse(ctx, winner); } throw err; } @@ -417,7 +564,12 @@ async function processUpload( await auditLog("documents.inbound.store", { userId: user.id, ipAddress: getClientIp(request), - details: { documentId: document.id, mime: detected.mimeType }, + details: { + documentId: document.id, + mime: detected.mimeType, + ...(scoped ? { scoped: true } : {}), + ...(sourceSystem ? { sourceSystem } : {}), + }, }); annotate({ @@ -428,6 +580,9 @@ async function processUpload( servingClass: detected.servingClass, linked: episodeIds.length, linkedVisits: encounterIds.length, + scoped, + sourceSystem, + aiDeferred: ctx.aiDeferred, }, }); @@ -437,7 +592,15 @@ async function processUpload( // enqueue is not awaited and swallows its own errors (a missing boss or a // transient send failure is a silent no-op). Only fresh inserts enqueue — a // duplicate upload returns early above and never reaches here. - void enqueueDocumentIndex(user.id, document.id); + // + // `aiRead=defer` narrows it to the local text layer: an import of a whole + // archive must not turn into one provider call per file. The person reads + // them later, deliberately, from the document itself. + if (ctx.aiDeferred) { + void enqueueDocumentIndex(user.id, document.id, { localOnly: true }); + } else { + void enqueueDocumentIndex(user.id, document.id); + } // Render a preview thumbnail in the background too (pure local compute — no // egress). Same fire-and-forget contract: the upload never blocks on or fails @@ -453,7 +616,12 @@ async function processUpload( // provider-free text-layer path. The persisted summary shows on the detail // view. Same fire-and-forget contract: the upload never blocks on or fails // because of it. Only fresh inserts reach here (a duplicate returns early). - const documentAi = await getAiCapability("documentAi"); + // + // A deferred upload skips it outright and keeps `summaryState: NONE`, so + // the detail sheet offers "Generate summary" rather than a pending state. + const documentAi = ctx.aiDeferred + ? { available: false as const, reason: "deferred" } + : await getAiCapability("documentAi"); if (documentAi.available) { void enqueueDocumentSummary(user.id, document.id); } else { @@ -463,6 +631,8 @@ async function processUpload( }); } + if (scoped) return receiptResponse(document.id, 201, { duplicate: false }); + const [links, visitLinks] = await Promise.all([ loadConditionLinks(user.id, [document.id]), loadDocumentEncounterLinks(user.id, [document.id]), @@ -481,9 +651,28 @@ async function processUpload( ); } +/** + * Find a document by its import source key, live or tombstoned. The blob + * column is never selected. + */ +function findBySourceKey( + userId: string, + sourceSystem: DocumentSourceSystemValue, + sourceId: string, +): Promise { + return prisma.inboundDocument.findFirst({ + where: { userId, sourceSystem, sourceId }, + omit: { contentEncrypted: true }, + }); +} + /** Authenticate and reserve memory capacity before any request-body read. */ async function postUpload(request: Request): Promise { - const { user } = await requireAuth(); + // The one route that names `documents:write`. Every other vault leg — + // list, detail, original, thumbnail, bulk, AI — declares no scope and so + // refuses the token (`bearer-scope-enforcement-guard.test.ts`). + const auth = await requireAuth(DOCUMENTS_WRITE_SCOPE); + const { user } = auth; const releaseUploadSlot = acquireDocumentUploadSlot(user.id); if (!releaseUploadSlot) { const response = apiError( @@ -499,7 +688,7 @@ async function postUpload(request: Request): Promise { } try { - return await processUpload(request, user); + return await processUpload(request, auth); } finally { releaseUploadSlot(); } @@ -517,7 +706,7 @@ export const POST = apiHandler(withIdempotency<[Request]>(postUpload)); * letter is health data belonging to the record, and reading it is the whole * point of handing somebody the record. * - * The POST above is NOT delegable and keeps `requireAuth()`. Uploading is not + * The POST above is NOT delegable and declares its own scope. Uploading is not * a verb the grant admits, and the module-level split matters here: the * resolver escalates any non-safe method to `"write"`, so the two arms have to * declare separately rather than share one line. diff --git a/src/app/api/tokens/documents/__tests__/route.test.ts b/src/app/api/tokens/documents/__tests__/route.test.ts new file mode 100644 index 000000000..d6cec1431 --- /dev/null +++ b/src/app/api/tokens/documents/__tests__/route.test.ts @@ -0,0 +1,220 @@ +/** + * The document-upload mint (#1038). + * + * The assertion that matters most is not the 201 — it is the exact + * `permissions` array handed to `issueApiToken`. That helper defaults to + * `["*"]` when the property is absent, so an edit that drops one line turns a + * user-facing endpoint into a cookie-equivalent credential factory, and a test + * that only checked the status code would go on passing. The structural guard + * catches a literal `permissions: ["*"` and would not catch the omission, so + * this is where that hole is closed. + */ +import { describe, it, expect, vi, beforeEach } from "vitest"; + +vi.mock("@/lib/api-handler", () => ({ + apiHandler: (fn: unknown) => fn, + requireCookieAuth: vi.fn(), +})); + +vi.mock("@/lib/auth/issue-token", () => ({ + issueApiToken: vi.fn(), +})); + +vi.mock("@/lib/app-settings", () => ({ + isApiGloballyEnabled: vi.fn(), +})); + +vi.mock("@/lib/rate-limit", () => ({ + checkRateLimit: vi.fn(), +})); + +vi.mock("@/lib/auth/audit", () => ({ + auditLog: vi.fn().mockResolvedValue(undefined), +})); + +vi.mock("@/lib/logging/context", () => ({ + annotate: vi.fn(), + getEvent: vi.fn(() => null), +})); + +vi.mock("@/lib/db", () => ({ + prisma: { apiToken: { count: vi.fn() } }, +})); + +import { POST } from "../route"; +import { requireCookieAuth } from "@/lib/api-handler"; +import { issueApiToken } from "@/lib/auth/issue-token"; +import { isApiGloballyEnabled } from "@/lib/app-settings"; +import { checkRateLimit } from "@/lib/rate-limit"; +import { auditLog } from "@/lib/auth/audit"; +import { prisma } from "@/lib/db"; +import { DOCUMENTS_WRITE_SCOPE } from "@/lib/documents/scopes"; + +const USER = { id: "user-1", role: "USER" as const }; +const EXPIRES = new Date("2027-08-31T00:00:00.000Z"); + +function req(body: unknown): Request { + return new Request("https://health.example/api/tokens/documents", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify(body), + }); +} + +beforeEach(() => { + vi.resetAllMocks(); + vi.mocked(requireCookieAuth).mockResolvedValue({ user: USER } as never); + vi.mocked(isApiGloballyEnabled).mockResolvedValue(true); + vi.mocked(checkRateLimit).mockResolvedValue({ allowed: true } as never); + vi.mocked(auditLog).mockResolvedValue(undefined as never); + // Below the ceiling by default; the cases that care set their own count. + vi.mocked(prisma.apiToken.count).mockResolvedValue(0 as never); + vi.mocked(issueApiToken).mockResolvedValue({ + token: "hlk_" + "a".repeat(64), + expiresAt: EXPIRES, + tokenId: "token-1", + name: "Paperless workflow", + } as never); +}); + +describe("what it mints", () => { + it("names exactly the upload scope, and no wildcard", async () => { + const res = await POST(req({ name: "Paperless workflow" }) as never); + expect(res.status).toBe(201); + + const opts = vi.mocked(issueApiToken).mock.calls[0][0]; + // Exact equality, not `toContain`: an extra scope is a widening, and this + // is the assertion standing between the endpoint and one. + expect(opts.permissions).toEqual([DOCUMENTS_WRITE_SCOPE]); + expect(opts.userId).toBe(USER.id); + }); + + it("returns the raw token once, with its name and expiry", async () => { + const res = await POST(req({ name: "Paperless workflow" }) as never); + const json = await res.json(); + expect(json.data.token).toMatch(/^hlk_[0-9a-f]{64}$/); + expect(json.data.name).toBe("Paperless workflow"); + expect(json.data.expiresAt).toBeDefined(); + }); + + it("expires by default rather than living forever", async () => { + await POST(req({ name: "Import script" }) as never); + const opts = vi.mocked(issueApiToken).mock.calls[0][0]; + expect(opts.expiresInDays).toBe(365); + }); + + it("honours an explicit lifetime", async () => { + await POST(req({ name: "Import script", expiresInDays: 30 }) as never); + expect(vi.mocked(issueApiToken).mock.calls[0][0].expiresInDays).toBe(30); + }); + + it("audits the mint with the token id and the scope", async () => { + await POST(req({ name: "Import script" }) as never); + expect(auditLog).toHaveBeenCalledWith( + "tokens.documents.create", + expect.objectContaining({ + userId: USER.id, + details: expect.objectContaining({ + tokenId: "token-1", + scope: DOCUMENTS_WRITE_SCOPE, + }), + }), + ); + }); + + it("counts only live tokens carrying this scope", async () => { + // The whole shape of the query, asserted rather than assumed. Counting + // every row would let the `["*"]` tokens a login mints consume the budget + // and refuse a mint for a reason the person cannot see; counting revoked + // or expired rows would mean revoking never frees a slot. + await POST(req({ name: "Import script" }) as never); + + const where = vi.mocked(prisma.apiToken.count).mock.calls[0][0]?.where; + expect(where).toMatchObject({ + userId: USER.id, + revoked: false, + permissions: { has: DOCUMENTS_WRITE_SCOPE }, + }); + expect(where?.OR).toEqual([ + { expiresAt: null }, + { expiresAt: { gt: expect.any(Date) } }, + ]); + }); +}); + +describe("when it refuses", () => { + it("403s while the operator has the API switched off", async () => { + vi.mocked(isApiGloballyEnabled).mockResolvedValue(false); + const res = await POST(req({ name: "Import script" }) as never); + expect(res.status).toBe(403); + expect(issueApiToken).not.toHaveBeenCalled(); + }); + + it("429s past the mint rate limit, without minting", async () => { + vi.mocked(checkRateLimit).mockResolvedValue({ allowed: false } as never); + const res = await POST(req({ name: "Import script" }) as never); + expect(res.status).toBe(429); + expect(issueApiToken).not.toHaveBeenCalled(); + }); + + it("422s on a missing name", async () => { + const res = await POST(req({}) as never); + expect(res.status).toBe(422); + expect(issueApiToken).not.toHaveBeenCalled(); + }); + + it("422s on a lifetime outside the permitted band", async () => { + const res = await POST( + req({ name: "Import script", expiresInDays: 4000 }) as never, + ); + expect(res.status).toBe(422); + expect(issueApiToken).not.toHaveBeenCalled(); + }); + + it("409s at the live-token ceiling, without minting", async () => { + vi.mocked(prisma.apiToken.count).mockResolvedValue(10 as never); + const res = await POST(req({ name: "Import script" }) as never); + + // 409, not 429: a conflict with the account's current state, whose remedy + // is revoking rather than waiting. A 429 would say the opposite. + expect(res.status).toBe(409); + expect((await res.json()).meta?.errorCode).toBe( + "tokens.documents.ceiling_reached", + ); + expect(issueApiToken).not.toHaveBeenCalled(); + }); + + it("still mints on the last free slot", async () => { + // The off-by-one that would make the ceiling nine. Worth pinning, because + // `>=` versus `>` is invisible in review and only one of them is right. + vi.mocked(prisma.apiToken.count).mockResolvedValue(9 as never); + const res = await POST(req({ name: "Import script" }) as never); + expect(res.status).toBe(201); + }); + + it("checks the ceiling only after the body validates", async () => { + // A malformed request should hear about the malformation, not the + // ceiling — and should not spend a query establishing it. + vi.mocked(prisma.apiToken.count).mockResolvedValue(10 as never); + const res = await POST(req({}) as never); + expect(res.status).toBe(422); + expect(prisma.apiToken.count).not.toHaveBeenCalled(); + }); + + it("ignores a scope the body tries to smuggle in", async () => { + // There is no `scope` field to validate, so an unknown key is simply not + // read — the permission array is a literal either way. Asserted because + // "the field does not exist" is the security property, and a schema change + // that added one would silently make this body meaningful. + await POST( + req({ + name: "Import script", + scope: "read_write", + permissions: ["*"], + }) as never, + ); + expect(vi.mocked(issueApiToken).mock.calls[0][0].permissions).toEqual([ + DOCUMENTS_WRITE_SCOPE, + ]); + }); +}); diff --git a/src/app/api/tokens/documents/route.ts b/src/app/api/tokens/documents/route.ts new file mode 100644 index 000000000..0a5121047 --- /dev/null +++ b/src/app/api/tokens/documents/route.ts @@ -0,0 +1,127 @@ +/** + * `POST /api/tokens/documents` — mint a Bearer that can upload documents into + * the vault and do nothing else (#1038). + * + * The door another document system pushes through: a Paperless-ngx workflow + * posting a newly tagged document, or the import script copying an archive + * across. It is the measurement-ingest mint (`../measurements/route.ts`) + * copied for a second scope, and every argument made there holds here — read + * that file for the reasoning behind each guard. In short: + * + * - `permissions` is a literal, so no request shape reaches it; + * - `documents:write` is accepted by exactly one route, the upload, on the + * holder's own record, and the answer it gets back is a receipt rather than + * the stored row — it cannot list, read, download, edit or delete a + * document, reach the AI legs, or mint another token; + * - minting takes a cookie session, because a short-lived native access token + * must not be able to leave behind a credential that lives a year; + * - ten mints a minute and ten live tokens bound a runaway loop. + * + * No listing and no revoke of its own: `GET /api/tokens` lists every token + * with its permissions and `DELETE /api/tokens/[id]` revokes one. + */ +import { NextRequest } from "next/server"; + +import { apiHandler, requireCookieAuth } from "@/lib/api-handler"; +import { + apiError, + apiSuccess, + getClientIp, + returnAllZodIssues, + safeJson, +} from "@/lib/api-response"; +import { annotate } from "@/lib/logging/context"; +import { auditLog } from "@/lib/auth/audit"; +import { prisma } from "@/lib/db"; +import { issueApiToken } from "@/lib/auth/issue-token"; +import { isApiGloballyEnabled } from "@/lib/app-settings"; +import { checkRateLimit } from "@/lib/rate-limit"; +import { DOCUMENTS_WRITE_SCOPE } from "@/lib/documents/scopes"; +import { createDocumentTokenSchema } from "@/lib/validations/tokens"; + +/** + * Days a token lives when the caller names no lifetime. A year, like the + * measurement token: it is pasted into a workflow that runs unattended, where + * a quiet expiry surfaces as "new letters stopped arriving in the spring". + */ +const DEFAULT_EXPIRY_DAYS = 365; + +/** Mints per user per minute. */ +const MINT_RATE_LIMIT_MAX = 10; +const MINT_RATE_LIMIT_WINDOW_MS = 60 * 1000; + +/** Live document tokens an account may hold at once; catches a loop. */ +const MAX_LIVE_TOKENS = 10; + +export const POST = apiHandler(async (request: NextRequest) => { + const { user } = await requireCookieAuth(); + annotate({ action: { name: "tokens.documents.create" } }); + + if (!(await isApiGloballyEnabled())) { + return apiError("API is globally disabled", 403); + } + + const rl = await checkRateLimit( + `tokens:documents:mint:${user.id}`, + MINT_RATE_LIMIT_MAX, + MINT_RATE_LIMIT_WINDOW_MS, + ); + if (!rl.allowed) { + return apiError("Too many token mints, try again later", 429); + } + + const { data: body, error: jsonError } = await safeJson(request, { + maxBytes: 16 * 1024, + }); + if (jsonError) return jsonError; + + const parsed = createDocumentTokenSchema.safeParse(body); + if (!parsed.success) { + return returnAllZodIssues(parsed.error, 422); + } + + // Counted live and per scope, after validation and before the create; the + // count-then-create race is bounded by the mint bucket above. See the + // measurement mint for the full argument. + const live = await prisma.apiToken.count({ + where: { + userId: user.id, + revoked: false, + permissions: { has: DOCUMENTS_WRITE_SCOPE }, + OR: [{ expiresAt: null }, { expiresAt: { gt: new Date() } }], + }, + }); + if (live >= MAX_LIVE_TOKENS) { + annotate({ + action: { name: "tokens.documents.create" }, + meta: { outcome: "ceiling_reached", live_token_count: live }, + }); + return apiError( + `You already have ${MAX_LIVE_TOKENS} document tokens. Revoke one before creating another.`, + 409, + { errorCode: "tokens.documents.ceiling_reached" }, + ); + } + + const issued = await issueApiToken({ + userId: user.id, + name: parsed.data.name, + // A literal, never spread and never derived from the body. `issueApiToken` + // defaults to `["*"]` when this property is absent, which is why the unit + // suite asserts this array and not merely the 201. + permissions: [DOCUMENTS_WRITE_SCOPE], + expiresInDays: parsed.data.expiresInDays ?? DEFAULT_EXPIRY_DAYS, + }); + + await auditLog("tokens.documents.create", { + userId: user.id, + ipAddress: getClientIp(request), + details: { tokenId: issued.tokenId, scope: DOCUMENTS_WRITE_SCOPE }, + }); + + // The raw token, once. It is stored as an HMAC and no path re-reveals it. + return apiSuccess( + { token: issued.token, name: issued.name, expiresAt: issued.expiresAt }, + 201, + ); +}); diff --git a/src/components/documents/__tests__/document-card.test.tsx b/src/components/documents/__tests__/document-card.test.tsx index 7b0aa82a1..8de53bd90 100644 --- a/src/components/documents/__tests__/document-card.test.tsx +++ b/src/components/documents/__tests__/document-card.test.tsx @@ -37,6 +37,8 @@ function doc(overrides: Partial = {}): InboundDocumentDto { lastIndexAttemptAt: null, lastIndexOutcome: null, hasThumbnail: false, + sourceSystem: null, + sourceId: null, createdAt: "2025-10-05T08:00:00.000Z", updatedAt: "2025-10-05T08:00:00.000Z", ...overrides, diff --git a/src/components/documents/__tests__/document-facts-review.test.tsx b/src/components/documents/__tests__/document-facts-review.test.tsx index 7ab4798ef..c2c64d314 100644 --- a/src/components/documents/__tests__/document-facts-review.test.tsx +++ b/src/components/documents/__tests__/document-facts-review.test.tsx @@ -89,6 +89,8 @@ function doc( lastIndexAttemptAt: null, lastIndexOutcome: null, hasThumbnail: false, + sourceSystem: null, + sourceId: null, createdAt: "2026-08-01T10:00:00.000Z", updatedAt: "2026-08-01T10:00:00.000Z", facts: [fact()], diff --git a/src/components/documents/__tests__/episode-documents-card.test.tsx b/src/components/documents/__tests__/episode-documents-card.test.tsx index db5856609..6df1c1dd7 100644 --- a/src/components/documents/__tests__/episode-documents-card.test.tsx +++ b/src/components/documents/__tests__/episode-documents-card.test.tsx @@ -57,6 +57,8 @@ function doc(id: string, title: string): InboundDocumentDto { lastIndexAttemptAt: null, lastIndexOutcome: null, hasThumbnail: false, + sourceSystem: null, + sourceId: null, createdAt: "2025-10-05T08:00:00.000Z", updatedAt: "2025-10-05T08:00:00.000Z", }; diff --git a/src/components/documents/__tests__/vault-utils.test.ts b/src/components/documents/__tests__/vault-utils.test.ts index 201963f85..ace2e807e 100644 --- a/src/components/documents/__tests__/vault-utils.test.ts +++ b/src/components/documents/__tests__/vault-utils.test.ts @@ -45,6 +45,8 @@ function doc(overrides: Partial = {}): InboundDocumentDto { lastIndexAttemptAt: null, lastIndexOutcome: null, hasThumbnail: false, + sourceSystem: null, + sourceId: null, createdAt: "2026-03-11T08:00:00.000Z", updatedAt: "2026-03-11T08:00:00.000Z", ...overrides, diff --git a/src/components/documents/document-detail-sheet.tsx b/src/components/documents/document-detail-sheet.tsx index 780105bfa..070fcec12 100644 --- a/src/components/documents/document-detail-sheet.tsx +++ b/src/components/documents/document-detail-sheet.tsx @@ -73,6 +73,7 @@ import { import { cn } from "@/lib/utils"; import { INBOUND_DOCUMENT_KINDS, + type DocumentSourceSystemValue, type DocumentSuggestionDto, type DocumentSummaryMode, type InboundDocumentDetailDto, @@ -698,6 +699,14 @@ export function DocumentDetailSheet({ : null; const title = doc?.title ?? doc?.filename ?? t("documents.card.untitled"); + // The product names are proper nouns and stay untranslated; only "another + // system" is prose. + const sourceSystemName = (system: DocumentSourceSystemValue): string => + system === "PAPERLESS" + ? "Paperless-ngx" + : system === "PAPRA" + ? "Papra" + : t("documents.detail.sourceOther"); const Icon = doc ? DOCUMENT_KIND_ICONS[doc.kind] : null; // v1.37.0 — the ONE eager fetch of the original, and only for a document // that is going to render it. @@ -1167,6 +1176,25 @@ export function DocumentDetailSheet({ })} {doc.filename ? ` · ${doc.filename}` : ""}

+ + {/* Where an imported document came from (#1038). Meta, so + muted like the line above; the id is what the person looks + up in the other system. */} + {doc.sourceSystem ? ( +

+ {doc.sourceId + ? t("documents.detail.importedFromWithId", { + system: sourceSystemName(doc.sourceSystem), + id: doc.sourceId, + }) + : t("documents.detail.importedFrom", { + system: sourceSystemName(doc.sourceSystem), + })} +

+ ) : null} ) : null} diff --git a/src/components/settings/api-section.tsx b/src/components/settings/api-section.tsx index 29ba7c2b7..c129059b6 100644 --- a/src/components/settings/api-section.tsx +++ b/src/components/settings/api-section.tsx @@ -6,10 +6,12 @@ import { Check, ChevronDown, Copy, + FileUp, Key, Loader2, Trash2, Upload, + type LucideIcon, } from "lucide-react"; import { toast } from "sonner"; @@ -55,6 +57,7 @@ export function ApiSection() {
+
); @@ -70,6 +73,64 @@ export function ApiSection() { * below is where credentials are reviewed and killed. */ function MeasurementTokensCard() { + const { t } = useTranslations(); + return ( + + ); +} + +/** + * Mint a token another document system uploads through (#1038): a Paperless + * workflow or the import script. Same card, same no-list reasoning as the + * measurement one above. + */ +function DocumentTokensCard() { + const { t } = useTranslations(); + return ( + + ); +} + +/** One single-scope mint: a name field, the token shown once, a copy button. */ +function IngestTokenCard({ + endpoint, + icon, + slot, + copy, +}: { + endpoint: "/api/tokens/measurements" | "/api/tokens/documents"; + icon: LucideIcon; + slot: string; + copy: { + title: string; + description: string; + detail: string; + scopeNote: string; + createFailed: string; + }; +}) { const { t } = useTranslations(); const queryClient = useQueryClient(); const [newName, setNewName] = useState(""); @@ -84,7 +145,7 @@ function MeasurementTokensCard() { setTokenMsg(null); setNewToken(null); try { - const res = await apiFetchRaw("/api/tokens/measurements", { + const res = await apiFetchRaw(endpoint, { method: "POST", headers: { "Content-Type": "application/json" }, // No scope field: the endpoint mints one shape, so there is nothing @@ -101,7 +162,7 @@ function MeasurementTokensCard() { // The server's own message when it sent one — it names the actual // cause (switch off, rate limit, validation) — and a message about // this specific action when it did not. - setTokenMsg(json.error || t("settings.measurementsToken.createFailed")); + setTokenMsg(json.error || copy.createFailed); } } catch { setTokenMsg(t("common.networkError")); @@ -125,17 +186,15 @@ function MeasurementTokensCard() { return (
-

- {t("settings.measurementsToken.detail")} -

+

{copy.detail}

- {t("settings.measurementsToken.scopeNote")} + {copy.scopeNote}

@@ -162,7 +221,7 @@ function MeasurementTokensCard() { {newToken && (

{t("settings.tokenCreated")} @@ -223,6 +282,12 @@ function ApiEndpointsCard() { auth: "Authorization: Bearer hlk_...", example: `{ "entries": [ { "hkIdentifier": "...", "value": 1, ... } ] }`, }, + { + method: "POST", + path: "/api/documents/inbound", + auth: "Authorization: Bearer hlk_...", + example: `multipart: file, title, documentDate, sourceSystem, sourceId`, + }, ]; return ( diff --git a/src/lib/data-wipe/wipe-plan.ts b/src/lib/data-wipe/wipe-plan.ts index e4d1b863b..a0e3ab52a 100644 --- a/src/lib/data-wipe/wipe-plan.ts +++ b/src/lib/data-wipe/wipe-plan.ts @@ -147,6 +147,11 @@ export const WIPE_MODELS = [ "DocumentContentIndex", "DocumentThumbnail", "DocumentConditionLink", + // The memory of which imported documents the person deleted (#1038). The + // documents themselves go below; keeping the memory would make a fresh + // import after the wipe skip files for a decision about a record that no + // longer exists. + "DocumentImportKey", "ExtractedFact", "ImportJob", diff --git a/src/lib/documents/__tests__/index-document.test.ts b/src/lib/documents/__tests__/index-document.test.ts index c9fab4ae9..40fb7409d 100644 --- a/src/lib/documents/__tests__/index-document.test.ts +++ b/src/lib/documents/__tests__/index-document.test.ts @@ -169,6 +169,23 @@ describe("indexDocumentContent — provider-first path", () => { }); describe("indexDocumentContent — local fallback path", () => { + it("a deferred import stays local even with an external pick and the toggle ON", async () => { + // #1038 — `aiRead=defer`: the person asked for AI reading to wait. + vi.mocked(resolveDocumentVisionProvider).mockResolvedValue(PICK as never); + visionOk(); + const outcome = await indexDocumentContent("user-1", "doc-1", { + localOnly: true, + }); + expect(outcome).toEqual({ + indexed: true, + source: "local-pdf", + tokenCount: 5, + }); + expect(resolveDocumentVisionProvider).not.toHaveBeenCalled(); + expect(transcribeDocument).not.toHaveBeenCalled(); + expect(reserveBudget).not.toHaveBeenCalled(); + }); + it("falls back to local when NO provider is configured", async () => { vi.mocked(resolveDocumentVisionProvider).mockResolvedValue({ chain: [], diff --git a/src/lib/documents/__tests__/upload-token-limit.test.ts b/src/lib/documents/__tests__/upload-token-limit.test.ts new file mode 100644 index 000000000..b263b5365 --- /dev/null +++ b/src/lib/documents/__tests__/upload-token-limit.test.ts @@ -0,0 +1,40 @@ +/** + * The per-token upload ceiling for `documents:write` callers (#1038) is an + * operator setting (`DOCUMENT_UPLOAD_LIMIT_PER_HOUR`), not a constant. Pins the + * default, the clamp, and the fallback on garbage — the same contract as + * `DOCUMENT_AI_LIMIT_PER_HOUR`. + */ +import { afterEach, describe, expect, it } from "vitest"; + +import { resolveDocumentUploadLimitPerHour } from "../upload-policy"; + +const ORIGINAL = process.env.DOCUMENT_UPLOAD_LIMIT_PER_HOUR; + +afterEach(() => { + if (ORIGINAL === undefined) delete process.env.DOCUMENT_UPLOAD_LIMIT_PER_HOUR; + else process.env.DOCUMENT_UPLOAD_LIMIT_PER_HOUR = ORIGINAL; +}); + +describe("resolveDocumentUploadLimitPerHour", () => { + it("defaults to 120 when unset", () => { + delete process.env.DOCUMENT_UPLOAD_LIMIT_PER_HOUR; + expect(resolveDocumentUploadLimitPerHour()).toBe(120); + }); + + it("honours an operator value", () => { + process.env.DOCUMENT_UPLOAD_LIMIT_PER_HOUR = "400"; + expect(resolveDocumentUploadLimitPerHour()).toBe(400); + }); + + it("falls back to the default on a non-number", () => { + process.env.DOCUMENT_UPLOAD_LIMIT_PER_HOUR = "lots"; + expect(resolveDocumentUploadLimitPerHour()).toBe(120); + }); + + it("clamps to 1-1000", () => { + process.env.DOCUMENT_UPLOAD_LIMIT_PER_HOUR = "0"; + expect(resolveDocumentUploadLimitPerHour()).toBe(1); + process.env.DOCUMENT_UPLOAD_LIMIT_PER_HOUR = "50000"; + expect(resolveDocumentUploadLimitPerHour()).toBe(1000); + }); +}); diff --git a/src/lib/documents/index-document.ts b/src/lib/documents/index-document.ts index a35e24ca7..faf8e984e 100644 --- a/src/lib/documents/index-document.ts +++ b/src/lib/documents/index-document.ts @@ -383,9 +383,22 @@ export async function indexLoadedDocument( export async function indexDocumentContent( userId: string, documentId: string, + options: { localOnly?: boolean } = {}, ): Promise { const document = await loadOwnedDocument(userId, documentId); if (!document) return { indexed: false, reason: "not-found" }; - const provider = await resolveIndexProvider(userId); + // v1.39.2 (#1038) — an import that asked for `aiRead=defer` gets the + // provider-free text layer only. An empty context is the same shape "no + // provider configured" already produces, so the tree falls straight to the + // local path without resolving a chain it would not use. + const provider: ResolvedIndexProvider = options.localOnly + ? { + chain: [], + pick: null, + consentOk: false, + dailyCap: 0, + costOwner: "operator", + } + : await resolveIndexProvider(userId); return indexLoadedDocument(userId, document, provider); } diff --git a/src/lib/documents/scopes.ts b/src/lib/documents/scopes.ts new file mode 100644 index 000000000..036d46fa2 --- /dev/null +++ b/src/lib/documents/scopes.ts @@ -0,0 +1,16 @@ +/** + * The Bearer scope for pushing documents into the vault from another system. + * + * A zero-import leaf for the reason `MEASUREMENTS_WRITE_SCOPE` is one (see + * `@/lib/measurements/scopes`): the mint route, the upload route, the settings + * card and the guards all read it, and none of them should inherit an import + * graph for a string. + * + * Deliberately alone. There is no read counterpart and must not be one: the + * list, detail, original, thumbnail, bulk and AI legs of the vault stay + * cookie-equivalent, so a credential pasted into a Paperless workflow or an + * import script can add a file and learn nothing about the vault it lands in. + */ + +/** The scope a narrow token must carry to upload a document. */ +export const DOCUMENTS_WRITE_SCOPE = "documents:write"; diff --git a/src/lib/documents/store.ts b/src/lib/documents/store.ts index 57c9cbbde..268504ed7 100644 --- a/src/lib/documents/store.ts +++ b/src/lib/documents/store.ts @@ -17,7 +17,10 @@ import type { InboundDocumentStatus, } from "@/generated/prisma/client"; import { servingClassFor } from "@/lib/documents/upload-policy"; -import { toIndexOutcome } from "@/lib/validations/inbound-documents"; +import { + toDocumentSourceSystem, + toIndexOutcome, +} from "@/lib/validations/inbound-documents"; import type { DocumentConditionLinkDto, DocumentEncounterLinkDto, @@ -247,6 +250,8 @@ export function serialiseDocument( ? null : toIndexOutcome(doc.lastIndexOutcome), hasThumbnail, + sourceSystem: toDocumentSourceSystem(doc.sourceSystem), + sourceId: doc.sourceSystem ? doc.sourceId : null, createdAt: doc.createdAt.toISOString(), updatedAt: doc.updatedAt.toISOString(), }; diff --git a/src/lib/documents/upload-policy.ts b/src/lib/documents/upload-policy.ts index 5a7ad1e92..55e114f5a 100644 --- a/src/lib/documents/upload-policy.ts +++ b/src/lib/documents/upload-policy.ts @@ -84,6 +84,32 @@ export function acquireDocumentUploadSlot(userId: string): (() => void) | null { }; } +/** + * Hourly upload ceiling for a narrow `documents:write` token, per token. + * + * Its own bucket rather than the person's: an overnight import running at this + * rate must never use up the allowance the web app and the phone draw on. The + * default is twice the cookie bucket because an importer is strictly serial + * and is the reason the token exists. Operator-tunable via + * `DOCUMENT_UPLOAD_LIMIT_PER_HOUR`, clamped to 1-1000 the way + * `DOCUMENT_AI_LIMIT_PER_HOUR` is, so a typo can neither shut the door nor + * remove the cap. The per-user quota, not this number, is the disk guard. + */ +const DOCUMENT_UPLOAD_TOKEN_LIMIT_DEFAULT = 120; +const DOCUMENT_UPLOAD_TOKEN_LIMIT_MIN = 1; +const DOCUMENT_UPLOAD_TOKEN_LIMIT_MAX = 1000; + +export function resolveDocumentUploadLimitPerHour(): number { + const raw = process.env.DOCUMENT_UPLOAD_LIMIT_PER_HOUR; + if (!raw) return DOCUMENT_UPLOAD_TOKEN_LIMIT_DEFAULT; + const parsed = Number.parseInt(raw, 10); + if (!Number.isFinite(parsed)) return DOCUMENT_UPLOAD_TOKEN_LIMIT_DEFAULT; + return Math.min( + DOCUMENT_UPLOAD_TOKEN_LIMIT_MAX, + Math.max(DOCUMENT_UPLOAD_TOKEN_LIMIT_MIN, parsed), + ); +} + export interface DocumentLimits { /** Per-file upload cap in bytes (admin-tunable, clamped to the ceiling). */ maxFileBytes: number; diff --git a/src/lib/export/backup-plan.ts b/src/lib/export/backup-plan.ts index b1e9a3a17..e84d1bb48 100644 --- a/src/lib/export/backup-plan.ts +++ b/src/lib/export/backup-plan.ts @@ -670,6 +670,8 @@ export const NOT_IN_BACKUP_MODELS: Readonly> = { "The record of a restore this host ran from one of its own stored copies. It names a backup row that exists only in this database, so carrying it would describe a restore the receiving host never performed.", OffhostBackupState: "When this host last put this account's copy in this operator's bucket, and how big it was. It describes one deployment's relationship with one bucket, so restoring it elsewhere would assert an off-host copy that host has never written.", + DocumentImportKey: + "The source keys of imported documents the person deleted and the purge then removed, kept so a re-run of the importer does not store them again. The backup carries only live documents, so the tombstones that hold the same memory before the purge do not travel either; carrying the ledger alone would restore half of one decision. After a restore, a deliberate re-import can bring such a document back, and the person deletes it again.", ImportJob: "A job record pointing at an uploaded file that the backup does not carry, so restoring it would resurrect a task with nothing to work on.", InviteToken: diff --git a/src/lib/export/records-backup.ts b/src/lib/export/records-backup.ts index 2948689e8..e1d6026c6 100644 --- a/src/lib/export/records-backup.ts +++ b/src/lib/export/records-backup.ts @@ -171,6 +171,12 @@ export interface DocumentBackupEntry { /** Refs #776 — the index-attempt record, carried verbatim (canonical DR). */ lastIndexAttemptAt?: string | null; lastIndexOutcome?: string | null; + /** + * v1.39.2 (#1038) — where an imported document came from, both payload + * kinds. Optional so a file written before the fields existed still parses. + */ + sourceSystem?: string | null; + sourceId?: string | null; } export interface RecordsBackupOptions { @@ -296,6 +302,8 @@ export async function buildRecordsBackupSection( summaryState: true, lastIndexAttemptAt: true, lastIndexOutcome: true, + sourceSystem: true, + sourceId: true, createdAt: true, updatedAt: true, }, @@ -314,6 +322,8 @@ export async function buildRecordsBackupSection( reportDate: true, documentDate: true, summaryEncrypted: true, + sourceSystem: true, + sourceId: true, createdAt: true, }, }); @@ -520,6 +530,8 @@ export async function buildRecordsBackupSection( documentDate: d.documentDate ? d.documentDate.toISOString().slice(0, 10) : null, + sourceSystem: d.sourceSystem, + sourceId: d.sourceId, createdAt: d.createdAt.toISOString(), }; diff --git a/src/lib/export/restore-backup.ts b/src/lib/export/restore-backup.ts index 596d60ab9..123e6c255 100644 --- a/src/lib/export/restore-backup.ts +++ b/src/lib/export/restore-backup.ts @@ -2019,6 +2019,10 @@ export async function restoreBackup( ? new Date(document.lastIndexAttemptAt) : null, lastIndexOutcome: document.lastIndexOutcome ?? null, + sourceSystem: document.sourceSystem ?? null, + sourceId: document.sourceSystem + ? (document.sourceId ?? null) + : null, createdAt: new Date(document.createdAt!), updatedAt: new Date(document.updatedAt!), })), diff --git a/src/lib/jobs/__tests__/document-index.test.ts b/src/lib/jobs/__tests__/document-index.test.ts index e175bc1a8..3d7652ae9 100644 --- a/src/lib/jobs/__tests__/document-index.test.ts +++ b/src/lib/jobs/__tests__/document-index.test.ts @@ -11,6 +11,9 @@ vi.mock("@/lib/documents/index-document", () => ({ })); vi.mock("@/lib/jobs/boss-instance", () => ({ getGlobalBoss: vi.fn() })); vi.mock("@/lib/logging/context", () => ({ annotate: vi.fn() })); +vi.mock("@/lib/documents/auto-stage-labs", () => ({ + maybeAutoStageLabFacts: vi.fn().mockResolvedValue(undefined), +})); import { DOCUMENT_INDEX_QUEUE, @@ -19,6 +22,7 @@ import { } from "../document-index"; import { indexDocumentContent } from "@/lib/documents/index-document"; import { getGlobalBoss } from "@/lib/jobs/boss-instance"; +import { maybeAutoStageLabFacts } from "@/lib/documents/auto-stage-labs"; beforeEach(() => { vi.clearAllMocks(); @@ -38,6 +42,15 @@ describe("enqueueDocumentIndex", () => { ); }); + it("carries localOnly for a deferred import, and only then", async () => { + const send = vi.fn().mockResolvedValue("job-1"); + vi.mocked(getGlobalBoss).mockReturnValue({ send } as never); + await enqueueDocumentIndex("user-1", "doc-1", { localOnly: true }); + await enqueueDocumentIndex("user-1", "doc-2"); + expect(send.mock.calls[0][1]).toMatchObject({ localOnly: true }); + expect(send.mock.calls[1][1]).not.toHaveProperty("localOnly"); + }); + it("no-ops when the boss is not running", async () => { vi.mocked(getGlobalBoss).mockReturnValue(null as never); const result = await enqueueDocumentIndex("user-1", "doc-1"); @@ -62,7 +75,27 @@ describe("runDocumentIndex", () => { tokenCount: 3, } as never); await runDocumentIndex({ userId: "user-1", documentId: "doc-1" }); - expect(indexDocumentContent).toHaveBeenCalledWith("user-1", "doc-1"); + expect(indexDocumentContent).toHaveBeenCalledWith("user-1", "doc-1", { + localOnly: false, + }); + expect(maybeAutoStageLabFacts).toHaveBeenCalledWith("user-1", "doc-1"); + }); + + it("a deferred import indexes locally and stages no lab facts", async () => { + vi.mocked(indexDocumentContent).mockResolvedValue({ + indexed: true, + source: "local-pdf", + tokenCount: 3, + } as never); + await runDocumentIndex({ + userId: "user-1", + documentId: "doc-1", + localOnly: true, + }); + expect(indexDocumentContent).toHaveBeenCalledWith("user-1", "doc-1", { + localOnly: true, + }); + expect(maybeAutoStageLabFacts).not.toHaveBeenCalled(); }); it("ignores a payload missing ids without calling the tree", async () => { diff --git a/src/lib/jobs/document-index.ts b/src/lib/jobs/document-index.ts index d6cbf4b4a..74f16d12d 100644 --- a/src/lib/jobs/document-index.ts +++ b/src/lib/jobs/document-index.ts @@ -26,6 +26,12 @@ export interface DocumentIndexPayload { userId: string; documentId: string; enqueuedAt?: string; + /** + * v1.39.2 (#1038) — set for an upload that asked for `aiRead=defer`: index + * from the local text layer only and stage no lab facts. Absent on every + * job enqueued before the field existed, which keeps their behaviour. + */ + localOnly?: boolean; } /** @@ -38,7 +44,10 @@ export async function runDocumentIndex( ): Promise { const { userId, documentId } = payload; if (!userId || !documentId) return; - const outcome = await indexDocumentContent(userId, documentId); + const localOnly = payload.localOnly === true; + const outcome = await indexDocumentContent(userId, documentId, { + localOnly, + }); annotate({ action: { name: "documents.autoIndex.run" }, meta: outcome.indexed @@ -51,7 +60,9 @@ export async function runDocumentIndex( // PENDING for the existing human review; nothing is committed. Fully gated + // idempotent inside; a failure here must never fail the index job, so it is // swallowed to a no-op (the manual extract button remains the fallback). - if (outcome.indexed) { + // A deferred import stages nothing: lab staging is an AI read, and the + // person asked for those to wait. + if (outcome.indexed && !localOnly) { await maybeAutoStageLabFacts(userId, documentId).catch(() => {}); } } @@ -67,6 +78,7 @@ export async function runDocumentIndex( export async function enqueueDocumentIndex( userId: string, documentId: string, + options: { localOnly?: boolean } = {}, ): Promise<{ enqueued: boolean }> { const boss = getGlobalBoss(); if (!boss) return { enqueued: false }; @@ -74,6 +86,7 @@ export async function enqueueDocumentIndex( userId, documentId, enqueuedAt: new Date().toISOString(), + ...(options.localOnly ? { localOnly: true } : {}), }; try { const jobId = await boss.send(DOCUMENT_INDEX_QUEUE, payload, { diff --git a/src/lib/jobs/document-purge.ts b/src/lib/jobs/document-purge.ts index eebb5eb7e..c63e211c0 100644 --- a/src/lib/jobs/document-purge.ts +++ b/src/lib/jobs/document-purge.ts @@ -7,7 +7,8 @@ * grace window — returning the TOAST space the encrypted blob holds (a * tombstoned document keeps counting against the quota until purged, so * "deleted" bytes are never invisible backup weight). Staged facts cascade - * via the FK. Restore-after-purge answers 409 at the API. + * via the FK. Restore-after-purge answers 409 at the API. An imported + * document's source key is carried to `DocumentImportKey` on the way out. */ import { type Job } from "pg-boss"; import { withBackgroundEvent } from "@/lib/logging/background"; @@ -38,10 +39,34 @@ export async function purgeExpiredDocumentTombstones( now: Date = new Date(), ): Promise { const cutoff = new Date(now.getTime() - DOCUMENT_PURGE_GRACE_DAYS * DAY_MS); - const { count } = await prisma.inboundDocument.deleteMany({ - where: { deletedAt: { not: null, lt: cutoff } }, + const expired = { deletedAt: { not: null, lt: cutoff } }; + return prisma.$transaction(async (tx) => { + // v1.39.2 (#1038) — an imported document's source key outlives its row. + // While the tombstone exists the key sits on it and the upload route + // answers "deleted"; once the row goes, this ledger is what keeps a re-run + // of the importer from storing the document again. Same transaction as the + // delete, so a purge that fails halfway forgets nothing. + const keyed = await tx.inboundDocument.findMany({ + where: { + ...expired, + sourceSystem: { not: null }, + sourceId: { not: null }, + }, + select: { userId: true, sourceSystem: true, sourceId: true }, + }); + if (keyed.length > 0) { + await tx.documentImportKey.createMany({ + data: keyed.map((row) => ({ + userId: row.userId, + sourceSystem: row.sourceSystem!, + sourceId: row.sourceId!, + })), + skipDuplicates: true, + }); + } + const { count } = await tx.inboundDocument.deleteMany({ where: expired }); + return count; }); - return count; } export async function handleDocumentPurge( diff --git a/src/lib/openapi/error-codes.ts b/src/lib/openapi/error-codes.ts index b3a1e4d1b..64548daa8 100644 --- a/src/lib/openapi/error-codes.ts +++ b/src/lib/openapi/error-codes.ts @@ -312,7 +312,10 @@ export const ERROR_CODE_CATALOGUE: Readonly> = "source-priority.body.invalid_json", "source-priority.body.invalid_shape", ], - tokens: ["tokens.measurements.ceiling_reached"], + tokens: [ + "tokens.documents.ceiling_reached", + "tokens.measurements.ceiling_reached", + ], vaccination: [ "vaccination.booster-invalid", "vaccination.booster-no-antigen", diff --git a/src/lib/openapi/routes/auth.ts b/src/lib/openapi/routes/auth.ts index a5c80ad19..1df96d83f 100644 --- a/src/lib/openapi/routes/auth.ts +++ b/src/lib/openapi/routes/auth.ts @@ -30,7 +30,10 @@ import { mfaWebauthnLoginVerifySchema, } from "@/lib/validations/mfa"; import { oidcNativeTokenSchema } from "@/lib/validations/oidc-native"; -import { createMeasurementTokenSchema } from "@/lib/validations/tokens"; +import { + createDocumentTokenSchema, + createMeasurementTokenSchema, +} from "@/lib/validations/tokens"; import { nativeHandoffTokenSchema } from "@/lib/validations/native-handoff"; import { stepUpMintSchema, @@ -436,6 +439,28 @@ const createMeasurementTokenResponse = z "The one response that carries a usable measurement-ingest Bearer value.", }); +const createDocumentTokenRequest = createDocumentTokenSchema.meta({ + id: "CreateDocumentTokenRequest", + description: + "Mint a document-upload Bearer. No `scope` field: the endpoint mints one shape. `name` is what the token is listed under; name it after the system it is pasted into. `expiresInDays` defaults to 365.", +}); + +const createDocumentTokenResponse = z + .object({ + token: z + .string() + .describe( + "The raw `hlk_` Bearer, RETURNED EXACTLY ONCE. It is stored only as an HMAC-SHA256 hash and cannot be retrieved later. Treat it as a secret.", + ), + name: z.string(), + expiresAt: z.iso.datetime({ offset: true }), + }) + .meta({ + id: "CreateDocumentTokenResponse", + description: + "The one response that carries a usable document-upload Bearer value.", + }); + // ── Registration, credential rotation, Codex device-auth ───────────── const registerRequest = registerSchema.meta({ @@ -1536,7 +1561,7 @@ export const authPaths: NonNullable = { summary: "List the caller's API tokens", description: "Every `ApiToken` row belonging to the caller, newest first, revoked ones included — presence in this list is not validity, `revoked` is. The hash is never returned and there is no path that re-reveals a token's plaintext.\n\n" + - "Three things the name understates. The list is not limited to tokens a person minted from the settings surface: a native login mints a wildcard access token as an `ApiToken` row, so a signed-in phone shows up here too. There is no GENERIC mint any more — the POST at this exact path issued `[\"medication:ingest\"]` and was removed, because that scope reached no ingest route while the pre-fail-closed default let it reach everything else. The credentials that work are minted where they are scoped: the per-medication API-endpoint toggle, `POST /api/mcp/tokens` for a connector, and the sibling `POST /api/tokens/measurements` for third-party measurement ingest. And unlike the revoke, this read is NOT gated on the operator's instance-wide API switch: that switch governs the surfaces a token is for, not a token's ability to authenticate, so tokens stay live while it is off and their owner has to be able to see them.", + "Three things the name understates. The list is not limited to tokens a person minted from the settings surface: a native login mints a wildcard access token as an `ApiToken` row, so a signed-in phone shows up here too. There is no GENERIC mint any more — the POST at this exact path issued `[\"medication:ingest\"]` and was removed, because that scope reached no ingest route while the pre-fail-closed default let it reach everything else. The credentials that work are minted where they are scoped: the per-medication API-endpoint toggle, `POST /api/mcp/tokens` for a connector, the sibling `POST /api/tokens/measurements` for third-party measurement ingest, and `POST /api/tokens/documents` for document upload from another system. And unlike the revoke, this read is NOT gated on the operator's instance-wide API switch: that switch governs the surfaces a token is for, not a token's ability to authenticate, so tokens stay live while it is off and their owner has to be able to see them.", responses: { "200": { description: "The caller's tokens, newest first.", @@ -1620,6 +1645,70 @@ export const authPaths: NonNullable = { }, }, }, + "/api/tokens/documents": { + post: { + tags: ["Auth"], + summary: "Mint a document-upload token", + description: + "Mints a Bearer scoped to exactly `documents:write` and audits the mint. THE RESPONSE CARRIES THE RAW TOKEN, once; it is stored as an HMAC and no path re-reveals it.\n\n" + + "**What it can do.** `POST /api/documents/inbound` on its owner's own record, and nothing else. It is the door another document system pushes through: a Paperless-ngx workflow posting a newly tagged document, or an import script copying an archive. Uploads through it draw on a bucket of their own, keyed on the token (default 120 an hour, operator-tunable via `DOCUMENT_UPLOAD_LIMIT_PER_HOUR`), and get back a receipt (`id`, `duplicate`, and `deleted` when the source key belongs to a document the owner deleted) rather than the stored row.\n\n" + + "**What it cannot do.** List, read, download, preview, edit, delete or bulk-act on a document, reach the document-AI routes, act on a shared record, or mint another token. Every other route names no scope and refuses it 403.\n\n" + + "Minting requires a COOKIE SESSION, for the reason the measurement mint gives: a day-lived native access token must not be able to leave behind a year-lived credential. Gated by the operator's instance-wide API switch. Body capped at 16 KiB; 10 mints per user per minute, and at most 10 live document tokens held at once. Tokens appear in `GET /api/tokens` and are revoked at `DELETE /api/tokens/{id}`.", + requestBody: { + required: true, + content: { + "application/json": { schema: createDocumentTokenRequest }, + }, + }, + responses: { + "201": { + description: + "Token minted. The `token` field is the only copy of the secret.", + content: { + "application/json": { + schema: dataEnvelope( + createDocumentTokenResponse, + "CreateDocumentTokenEnvelope", + ), + }, + }, + }, + "400": { + description: "Body is not parseable JSON.", + content: { "application/json": { schema: errorEnvelope } }, + }, + "403": { + description: + "The operator has switched the API off instance-wide. Nothing was minted. A Bearer caller is refused 401 before the switch is read.", + content: { "application/json": { schema: errorEnvelope } }, + }, + "413": { + description: "Body exceeds 16 KiB.", + content: { "application/json": { schema: errorEnvelope } }, + }, + "415": { + description: "Content-Type is not `application/json`.", + content: { "application/json": { schema: errorEnvelope } }, + }, + ...stdResponses, + "422": { + description: + "Validation failed. The envelope carries every offending issue.", + content: { "application/json": { schema: errorEnvelope } }, + }, + "429": { + description: + "More than 10 mints from this account in a minute. Nothing was minted.", + content: { "application/json": { schema: errorEnvelope } }, + }, + "409": { + description: + "The account already holds 10 live document tokens (`meta.errorCode` = `tokens.documents.ceiling_reached`). Nothing was minted. Revoke one at `DELETE /api/tokens/{id}`; only live tokens carrying `documents:write` count.", + content: { "application/json": { schema: errorEnvelope } }, + }, + }, + }, + }, "/api/tokens/{id}": { delete: { tags: ["Auth"], diff --git a/src/lib/openapi/routes/documents.ts b/src/lib/openapi/routes/documents.ts index 3ee9bbb2a..3f99bf02f 100644 --- a/src/lib/openapi/routes/documents.ts +++ b/src/lib/openapi/routes/documents.ts @@ -23,6 +23,8 @@ import { documentUpdateSchema, inboundConfirmSchema, inboundFactEditSchema, + DOCUMENT_SOURCE_ID_MAX, + DOCUMENT_SOURCE_SYSTEMS, DOCUMENT_SUMMARY_STATES, INBOUND_DOCUMENT_KINDS, INBOUND_DOCUMENT_STATUSES, @@ -204,13 +206,15 @@ const inboundDocument = z ]) .nullable(), hasThumbnail: z.boolean(), + sourceSystem: z.enum(DOCUMENT_SOURCE_SYSTEMS).nullable(), + sourceId: z.string().nullable(), createdAt: z.string(), updatedAt: z.string(), }) .meta({ id: "InboundDocument", description: - "A stored document. The raw bytes are stored encrypted at rest and never returned; this is the metadata + staging summary. `status` is STORED for a freshly uploaded file (no extraction run). `title` is the user label (plaintext); `documentDate` is the user filing date; `reportDate` is the model-transcribed date (null until extraction runs). `servingClass` says how `/original` delivers the file — render inline (`inline`) or download-only (`attachment`); render/download by it, never by MIME guess. `hasContentIndex` is true when the document has a content-search index (auto-indexed on upload); `contentIndexSource` says how that index was produced — `vision` means an AI provider read the original, the other values are provider-free local extractions, and it is null when `hasContentIndex` is false. `lastIndexAttemptAt` is when the most recent index attempt finished (successful or not; null when none ran); `lastIndexOutcome` says why that attempt produced no index, and is null when the attempt succeeded or none ran. `hasThumbnail` is true when a preview thumbnail has been rendered in the background (fetch it from `/api/documents/inbound/{id}/thumbnail`); false means no preview yet or an unsupported type — show a placeholder. Treat an unknown `kind` value as OTHER when decoding.", + "A stored document. The raw bytes are stored encrypted at rest and never returned; this is the metadata + staging summary. `status` is STORED for a freshly uploaded file (no extraction run). `title` is the user label (plaintext); `documentDate` is the user filing date; `reportDate` is the model-transcribed date (null until extraction runs). `servingClass` says how `/original` delivers the file — render inline (`inline`) or download-only (`attachment`); render/download by it, never by MIME guess. `hasContentIndex` is true when the document has a content-search index (auto-indexed on upload); `contentIndexSource` says how that index was produced — `vision` means an AI provider read the original, the other values are provider-free local extractions, and it is null when `hasContentIndex` is false. `lastIndexAttemptAt` is when the most recent index attempt finished (successful or not; null when none ran); `lastIndexOutcome` says why that attempt produced no index, and is null when the attempt succeeded or none ran. `hasThumbnail` is true when a preview thumbnail has been rendered in the background (fetch it from `/api/documents/inbound/{id}/thumbnail`); false means no preview yet or an unsupported type — show a placeholder. `sourceSystem` / `sourceId` say where an imported document came from and its id there (both null for a document added in HealthLog); treat an unknown `sourceSystem` as OTHER when decoding. Treat an unknown `kind` value as OTHER when decoding.", }); const inboundDocumentDetail = inboundDocument @@ -262,11 +266,23 @@ const confirmResponse = z const kindValues = [...INBOUND_DOCUMENT_KINDS]; +const documentUploadReceipt = z + .object({ + id: z.string().nullable(), + duplicate: z.boolean(), + deleted: z.literal(true).optional(), + }) + .meta({ + id: "DocumentUploadReceipt", + description: + "What an upload answers instead of the stored row. A narrow `documents:write` token always gets this shape (`duplicate: false` on 201, `true` on 200): a credential that can only add files learns whether a file is stored, not its title, filename or links. Every caller gets it when the upload's source key belongs to a document the owner DELETED (`deleted: true`, 200, nothing stored); `id` is the tombstoned row, or null once the 30-day purge removed it.", + }); + // Shared by the POST 200 (duplicate) and 201 (created) responses — the // envelope id must be registered exactly once. -const inboundDocumentEnvelope = dataEnvelope( - inboundDocument, - "InboundDocumentEnvelope", +const documentUploadEnvelope = dataEnvelope( + z.union([inboundDocument, documentUploadReceipt]), + "DocumentUploadEnvelope", ); export const inboundDocumentPaths: NonNullable = { @@ -387,7 +403,9 @@ export const inboundDocumentPaths: NonNullable = { tags: ["Documents"], summary: "Store a document (no extraction)", description: - 'STORE-ONLY upload. Stores the raw document ENCRYPTED at rest with `status: STORED` and runs NO extraction — provider-free, no AI consent / budget / egress. A file is always filable, even with no document-scan provider configured. `multipart/form-data`: a `file` plus optional `title`, `kind`, `documentDate` (YYYY-MM-DD), and repeated `episodeIds` form fields (pre-link to the caller\'s illness/condition episodes). Accepted types (magic-byte sniffed, never the wire Content-Type): PDF/JPEG/PNG/WebP/GIF render inline; Office (docx/xlsx/pptx/doc/xls/ppt), text/CSV/Markdown/RTF, TIFF, HEIC/HEIF, XML/JSON are stored verbatim and served download-only. HEIC is stored as-is but attachment-only — prefer transcoding to JPEG client-side for inline preview parity. Error contract: `413` with `meta.reason = "fileTooLarge"` (+ `maxFileBytes`) or `"quotaExceeded"` (+ `quotaBytes`, `usedBytes`); `415` with `meta.reason = "unsupportedType"`. A same-user duplicate (same bytes) returns 200 + `meta.duplicate: true` with the existing row — not an error. `Idempotency-Key` honoured. Read `GET /api/documents/inbound/usage` for the effective limits before offering an upload. AI extraction is a separate opt-in action — see `POST /api/documents/inbound/{id}/extract`.', + 'STORE-ONLY upload. Stores the raw document ENCRYPTED at rest with `status: STORED` and runs NO extraction — provider-free, no AI consent / budget / egress. A file is always filable, even with no document-scan provider configured. `multipart/form-data`: a `file` plus optional `title`, `kind`, `documentDate` (YYYY-MM-DD), and repeated `episodeIds` form fields (pre-link to the caller\'s illness/condition episodes). Accepted types (magic-byte sniffed, never the wire Content-Type): PDF/JPEG/PNG/WebP/GIF render inline; Office (docx/xlsx/pptx/doc/xls/ppt), text/CSV/Markdown/RTF, TIFF, HEIC/HEIF, XML/JSON are stored verbatim and served download-only. HEIC is stored as-is but attachment-only — prefer transcoding to JPEG client-side for inline preview parity. Error contract: `413` with `meta.reason = "fileTooLarge"` (+ `maxFileBytes`) or `"quotaExceeded"` (+ `quotaBytes`, `usedBytes`); `415` with `meta.reason = "unsupportedType"`. A same-user duplicate (same bytes) returns 200 + `meta.duplicate: true` with the existing row — not an error. `Idempotency-Key` honoured. Read `GET /api/documents/inbound/usage` for the effective limits before offering an upload. AI extraction is a separate opt-in action — see `POST /api/documents/inbound/{id}/extract`.\n\n' + + "**Importing from another system (v1.39.2).** Optional `sourceSystem` (`PAPERLESS`, `PAPRA`, `OTHER`) and `sourceId` (its id there, printable, up to 128 characters; needs `sourceSystem`) key the upload. A re-send with a known key is a duplicate: 200 with the live row (`meta.duplicate: true`), or, when the owner DELETED that document, 200 with `data.deleted: true` and `meta.deleted: true` and nothing stored; this holds after the 30-day purge too. `aiRead=defer` holds back automatic AI reading for this upload: the thumbnail and a local text index still run, the summary and lab staging do not, so the document waits for the person to read it deliberately. Absent, behaviour is unchanged.\n\n" + + "**Narrow token.** Also reachable with a `documents:write` Bearer (minted at `POST /api/tokens/documents`), and it is the only route that scope reaches. Such a caller draws on a bucket of its own, keyed on the token (default 120 an hour, `DOCUMENT_UPLOAD_LIMIT_PER_HOUR`, clamped 1-1000; the 429 carries `Retry-After` and `X-RateLimit-*`), and gets a `DocumentUploadReceipt` instead of the stored row. A cookie or wildcard caller keeps the 60-an-hour per-user bucket and the full row. With the vault module off the answer is 403 `module.disabled` for every caller.", requestBody: { required: true, content: { @@ -417,6 +435,18 @@ export const inboundDocumentPaths: NonNullable = { description: "Optional visit pre-links (repeat the field per id). Every id must be a live visit of the caller; an unknown one refuses the upload rather than dropping the link the person asked for.", }), + sourceSystem: z.enum(DOCUMENT_SOURCE_SYSTEMS).optional().meta({ + description: + "Optional. The system an imported document comes from. Shown on the detail view.", + }), + sourceId: z.string().max(DOCUMENT_SOURCE_ID_MAX).optional().meta({ + description: + "Optional. The document's id in `sourceSystem`; requires it. Unique per account across live and deleted documents.", + }), + aiRead: z.enum(["defer"]).optional().meta({ + description: + "Optional. `defer` holds back automatic AI reading for this upload (local index and thumbnail only).", + }), }), }, }, @@ -425,18 +455,19 @@ export const inboundDocumentPaths: NonNullable = { ...idempotentWrite(), "200": { description: - "Duplicate — the same bytes are already stored: the existing row, with `meta.duplicate: true` at the envelope level.", + "Duplicate — the same bytes, or the same source key, are already stored: the existing row (a `DocumentUploadReceipt` for a `documents:write` token), with `meta.duplicate: true` at the envelope level. When the source key belongs to a document the owner deleted: a receipt with `deleted: true` and `meta.deleted: true`, nothing stored.", content: { "application/json": { - schema: inboundDocumentEnvelope, + schema: documentUploadEnvelope, }, }, }, "201": { - description: "The stored document.", + description: + "The stored document (a `DocumentUploadReceipt` for a `documents:write` token).", content: { "application/json": { - schema: inboundDocumentEnvelope, + schema: documentUploadEnvelope, }, }, }, diff --git a/src/lib/validations/backup.ts b/src/lib/validations/backup.ts index d7abc0d74..088f066d7 100644 --- a/src/lib/validations/backup.ts +++ b/src/lib/validations/backup.ts @@ -81,6 +81,10 @@ import { emergencyBloodTypeSchema, organDonorStatusSchema, } from "@/lib/validations/emergency-profile"; +import { + DOCUMENT_SOURCE_ID_MAX, + DOCUMENT_SOURCE_SYSTEMS, +} from "@/lib/validations/inbound-documents"; import { REMINDER_EVENT_SOURCES } from "@/lib/measurement-reminders/satisfy"; import { UNITS_PER_DOSE_MESSAGE, @@ -1621,6 +1625,10 @@ const documentBackupSchema = z // Free string on purpose: a future outcome value must not fail a restore. lastIndexAttemptAt: isoDateTime.nullable().optional(), lastIndexOutcome: z.string().nullable().optional(), + // v1.39.2 (#1038) — import provenance. Closed like the upload field, so a + // restore cannot write a system the detail sheet has no name for. + sourceSystem: z.enum(DOCUMENT_SOURCE_SYSTEMS).nullable().optional(), + sourceId: z.string().max(DOCUMENT_SOURCE_ID_MAX).nullable().optional(), summary: z.string().nullable().optional(), createdAt: isoDateTime.optional(), updatedAt: isoDateTime.optional(), diff --git a/src/lib/validations/inbound-documents.ts b/src/lib/validations/inbound-documents.ts index 669c55f91..8e7d6f7bd 100644 --- a/src/lib/validations/inbound-documents.ts +++ b/src/lib/validations/inbound-documents.ts @@ -441,6 +441,12 @@ export interface InboundDocumentDto { * shows its kind icon instead of fetching a thumbnail that 404s. */ hasThumbnail: boolean; + /** + * v1.39.2 (#1038) — the system an imported document came from, or null for + * one added in HealthLog. Paired with `sourceId`, its id over there. + */ + sourceSystem: DocumentSourceSystemValue | null; + sourceId: string | null; createdAt: string; updatedAt: string; } @@ -669,13 +675,59 @@ const vaccinationIdList = z * becoming a second trip. No `userId` field; it is always narrowed from the * session. */ -export const documentCreateSchema = z.object({ - title: z.string().trim().min(1).max(DOCUMENT_TITLE_MAX).optional(), - kind: z.enum(INBOUND_DOCUMENT_KINDS).optional(), - documentDate: isoDateString.optional(), - episodeIds: episodeIdList.optional(), - encounterIds: encounterIdList.optional(), -}); +/** + * v1.39.2 (#1038) — the systems an imported document can say it came from. + * Closed on purpose: the detail sheet names each one, and a free string here + * would be a label the person never chose rendered as if HealthLog vouched for + * it. OTHER covers any importer that is neither. + */ +export const DOCUMENT_SOURCE_SYSTEMS = ["PAPERLESS", "PAPRA", "OTHER"] as const; +export type DocumentSourceSystemValue = + (typeof DOCUMENT_SOURCE_SYSTEMS)[number]; + +/** Narrow a stored `sourceSystem` string to the closed set (null otherwise). */ +export function toDocumentSourceSystem( + value: string | null | undefined, +): DocumentSourceSystemValue | null { + return (DOCUMENT_SOURCE_SYSTEMS as readonly string[]).includes(value ?? "") + ? (value as DocumentSourceSystemValue) + : null; +} + +/** Max length of an imported document's id in its source system. */ +export const DOCUMENT_SOURCE_ID_MAX = 128; + +export const documentCreateSchema = z + .object({ + title: z.string().trim().min(1).max(DOCUMENT_TITLE_MAX).optional(), + kind: z.enum(INBOUND_DOCUMENT_KINDS).optional(), + documentDate: isoDateString.optional(), + episodeIds: episodeIdList.optional(), + encounterIds: encounterIdList.optional(), + /** Where an imported document came from (provenance, shown on detail). */ + sourceSystem: z.enum(DOCUMENT_SOURCE_SYSTEMS).optional(), + /** + * The document's id in `sourceSystem`. Printable ASCII only: it is a key, + * not prose, and it is echoed on the detail sheet. + */ + sourceId: z + .string() + .trim() + .min(1) + .max(DOCUMENT_SOURCE_ID_MAX) + .regex(/^[\x21-\x7e]+$/u, "Expected printable characters without spaces") + .optional(), + /** + * `defer` holds back automatic AI reading for this upload: the thumbnail + * and a local text index still run, the summary and the lab staging do + * not. Absent means today's behaviour. + */ + aiRead: z.enum(["defer"]).optional(), + }) + .refine((v) => v.sourceId === undefined || v.sourceSystem !== undefined, { + path: ["sourceSystem"], + message: "sourceId needs a sourceSystem", + }); export type DocumentCreateInput = z.infer; diff --git a/src/lib/validations/tokens.ts b/src/lib/validations/tokens.ts index 5d475e499..d6e2ae03f 100644 --- a/src/lib/validations/tokens.ts +++ b/src/lib/validations/tokens.ts @@ -24,3 +24,13 @@ export const createMeasurementTokenSchema = z.object({ export type CreateMeasurementTokenInput = z.infer< typeof createMeasurementTokenSchema >; + +/** + * Body of `POST /api/tokens/documents` — mint a Bearer that can upload + * documents and nothing else. The same shape as the measurement mint and for + * the same reasons: no scope field, a bounded lifetime. + */ +export const createDocumentTokenSchema = z.object({ + name: z.string().min(1, "Name required").max(100), + expiresInDays: z.number().int().min(1).max(365).optional(), +}); diff --git a/tests/integration/backup-round-trip.test.ts b/tests/integration/backup-round-trip.test.ts index 2a04eddb3..e262a746a 100644 --- a/tests/integration/backup-round-trip.test.ts +++ b/tests/integration/backup-round-trip.test.ts @@ -823,6 +823,10 @@ async function seedEveryTwoEndedModel(prisma: PrismaClient): Promise { byteSize: documentBytes.byteLength, contentEncrypted, contentCodec: "binary2", + // Imported (#1038): the source key has to survive the trip, or a + // re-run of the importer after a restore stores the page twice. + sourceSystem: "PAPERLESS", + sourceId: "4711", }, }); @@ -1505,6 +1509,10 @@ describe("every model the plan claims two-ended survives a real restore", () => const vaultDocument = await prisma.inboundDocument.findFirstOrThrow({ where: { userId: OWNER_ID }, }); + expect({ + sourceSystem: vaultDocument.sourceSystem, + sourceId: vaultDocument.sourceId, + }).toEqual({ sourceSystem: "PAPERLESS", sourceId: "4711" }); const threads = await prisma.coachConversation.findMany({ where: { userId: OWNER_ID }, orderBy: { createdAt: "asc" }, diff --git a/tests/integration/documents-import.test.ts b/tests/integration/documents-import.test.ts new file mode 100644 index 000000000..83f8d830d --- /dev/null +++ b/tests/integration/documents-import.test.ts @@ -0,0 +1,461 @@ +/** + * Document import (#1038) against a real Postgres, the real resolver and the + * real upload route — no mocked Prisma, no mocked `requireAuth`. + * + * 1. A `documents:write` token minted from a cookie session uploads and gets + * a receipt; a re-send is a duplicate; after the owner deletes the + * document a re-send is answered "deleted" and stores nothing; after the + * purge the ledger keeps answering that. + * 2. The source key's unique index covers tombstones at the database level. + * 3. The token's own bucket 429s with Retry-After without touching the + * owner's; the quota answers 413 with its figures. + * 4. The token reaches no other route: every other vault leg, the mint, and + * the measurement write refuse it. + * 5. With the vault module off the token gets 403 `module.disabled`. + */ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { NextRequest } from "next/server"; + +process.env.ENCRYPTION_KEY ??= + "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"; +process.env.API_TOKEN_HMAC_KEY ??= + "test-hmac-key-documents-import-32-bytes-min-0987654321abcd"; + +import { cookieJar, headerJar } from "./mock-next-headers"; +import { getPrismaClient, truncateAllTables } from "./setup"; + +vi.mock("next/headers", async () => { + const { cookieJar, headerJar } = await import("./mock-next-headers"); + return { + headers: vi.fn(async () => ({ + get: (name: string) => headerJar.get(name.toLowerCase()) ?? null, + })), + cookies: vi.fn(async () => ({ + get: (name: string) => { + const value = cookieJar.get(name); + return value ? { name, value } : undefined; + }, + set: (name: string, value: string) => { + cookieJar.set(name, value); + }, + delete: (name: string) => { + cookieJar.delete(name); + }, + })), + }; +}); + +vi.mock("@/lib/db-compat", () => ({ + ensureDbCompatibility: vi.fn().mockResolvedValue(undefined), +})); + +const PNG_1X1 = Buffer.from( + "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR4nGNgYGBgAAAABQABXvMqOgAAAABJRU5ErkJggg==", + "base64", +); +const pdf = (tag: string) => + Buffer.from(`%PDF-1.7\n% ${tag}\n1 0 obj\n<<>>\nendobj\n%%EOF\n`); + +let userId = ""; +let sessionId = ""; +let token = ""; + +async function seedUser(modules: Record) { + const prisma = getPrismaClient(); + const user = await prisma.user.create({ + data: { + username: "importer", + email: "importer@example.test", + role: "USER", + timezone: "UTC", + modulePreferencesJson: modules, + }, + }); + const session = await prisma.session.create({ + data: { userId: user.id, expiresAt: new Date(Date.now() + 600_000) }, + }); + userId = user.id; + sessionId = session.id; +} + +function asCookie() { + headerJar.clear(); + cookieJar.set("healthlog_session", sessionId); +} + +function asToken() { + cookieJar.clear(); + headerJar.set("authorization", `Bearer ${token}`); +} + +async function mintDocumentToken(): Promise { + asCookie(); + const { POST } = await import("@/app/api/tokens/documents/route"); + const res = await POST( + new NextRequest("https://health.example/api/tokens/documents", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ name: "Paperless workflow" }), + } as never), + ); + expect(res.status).toBe(201); + return (await res.json()).data.token as string; +} + +function upload( + bytes: Buffer, + fields: Record = {}, + filename = "letter.pdf", +): Request { + const payload = new Uint8Array(bytes.byteLength); + payload.set(bytes); + const form = new FormData(); + form.append("file", new Blob([payload]), filename); + for (const [k, v] of Object.entries(fields)) form.append(k, v); + const headers: Record = {}; + const auth = headerJar.get("authorization"); + if (auth) headers.authorization = auth; + return new Request("http://localhost/api/documents/inbound", { + method: "POST", + body: form, + headers, + }); +} + +async function post(r: Request): Promise { + const route = await import("@/app/api/documents/inbound/route"); + return (route.POST as unknown as (r: Request) => Promise)(r); +} + +type Ctx = { params: Promise<{ id: string }> }; +const ctx = (id: string): Ctx => ({ params: Promise.resolve({ id }) }); + +beforeEach(async () => { + await truncateAllTables(getPrismaClient()); + cookieJar.clear(); + headerJar.clear(); + delete process.env.DOCUMENT_UPLOAD_LIMIT_PER_HOUR; +}); + +afterEach(() => { + delete process.env.DOCUMENT_UPLOAD_LIMIT_PER_HOUR; +}); + +describe("document import through a documents:write token", () => { + it("201 → 200 duplicate → delete → 200 deleted, no new row → purge → still deleted", async () => { + await seedUser({ inboundDocuments: true }); + token = await mintDocumentToken(); + const prisma = getPrismaClient(); + + // The mint wrote exactly one narrow scope. + const row = await prisma.apiToken.findFirstOrThrow({ where: { userId } }); + expect(row.permissions).toEqual(["documents:write"]); + + const fields = { + title: "Befund Kardiologie", + documentDate: "2019-03-05", + sourceSystem: "PAPERLESS", + sourceId: "412", + aiRead: "defer", + }; + + asToken(); + const first = await post(upload(pdf("a"), fields)); + expect(first.status).toBe(201); + const receipt = await first.json(); + // A receipt, not the row: no title, filename, kind or links. + expect(Object.keys(receipt.data).sort()).toEqual(["duplicate", "id"]); + expect(receipt.data.duplicate).toBe(false); + const id = receipt.data.id as string; + + const stored = await prisma.inboundDocument.findUniqueOrThrow({ + where: { id }, + omit: { contentEncrypted: true }, + }); + expect(stored).toMatchObject({ + userId, + title: "Befund Kardiologie", + sourceSystem: "PAPERLESS", + sourceId: "412", + summaryState: "NONE", + }); + expect(stored.documentDate?.toISOString().slice(0, 10)).toBe("2019-03-05"); + + // Re-send: same source key → duplicate, even with different bytes. + const again = await post(upload(pdf("a-rescanned"), fields)); + expect(again.status).toBe(200); + expect((await again.json()).data).toEqual({ id, duplicate: true }); + + // The owner sees where it came from. + asCookie(); + const byId = await import("@/app/api/documents/inbound/[id]/route"); + const detail = await ( + byId.GET as unknown as (r: Request, c: Ctx) => Promise + )(new Request(`http://localhost/api/documents/inbound/${id}`), ctx(id)); + expect(detail.status).toBe(200); + expect((await detail.json()).data).toMatchObject({ + sourceSystem: "PAPERLESS", + sourceId: "412", + }); + + // The owner deletes it (tombstone). + const del = await ( + byId.DELETE as unknown as (r: Request, c: Ctx) => Promise + )( + new Request(`http://localhost/api/documents/inbound/${id}`, { + method: "DELETE", + }), + ctx(id), + ); + expect(del.status).toBe(200); + + // A re-run of the importer does not bring it back. + asToken(); + const afterDelete = await post(upload(pdf("a"), fields)); + expect(afterDelete.status).toBe(200); + const deletedBody = await afterDelete.json(); + expect(deletedBody.data).toEqual({ id, duplicate: true, deleted: true }); + expect(deletedBody.meta).toEqual({ duplicate: true, deleted: true }); + expect(await prisma.inboundDocument.count({ where: { userId } })).toBe(1); + + // Past the grace window the purge takes the row and keeps the key. + const { purgeExpiredDocumentTombstones } = + await import("@/lib/jobs/document-purge"); + const purged = await purgeExpiredDocumentTombstones( + prisma as never, + new Date(Date.now() + 31 * 86_400_000), + ); + expect(purged).toBe(1); + expect(await prisma.inboundDocument.count({ where: { userId } })).toBe(0); + expect( + await prisma.documentImportKey.findMany({ + where: { userId }, + select: { sourceSystem: true, sourceId: true }, + }), + ).toEqual([{ sourceSystem: "PAPERLESS", sourceId: "412" }]); + + const afterPurge = await post(upload(pdf("a"), fields)); + expect(afterPurge.status).toBe(200); + expect((await afterPurge.json()).data).toEqual({ + id: null, + duplicate: true, + deleted: true, + }); + expect(await prisma.inboundDocument.count({ where: { userId } })).toBe(0); + + // A different source key is a new document. + const other = await post(upload(pdf("b"), { ...fields, sourceId: "413" })); + expect(other.status).toBe(201); + }); + + it("the source-key index holds tombstones at the database level", async () => { + await seedUser({ inboundDocuments: true }); + const prisma = getPrismaClient(); + const base = { + userId, + mimeType: "application/pdf", + byteSize: 1, + contentEncrypted: new Uint8Array([1]), + sourceSystem: "PAPRA", + sourceId: "doc_1", + }; + await prisma.inboundDocument.create({ + data: { ...base, deletedAt: new Date() }, + }); + await expect( + prisma.inboundDocument.create({ data: base }), + ).rejects.toMatchObject({ code: "P2002" }); + // Rows without a source key are not constrained by it. + await prisma.inboundDocument.create({ + data: { ...base, sourceSystem: null, sourceId: null }, + }); + await prisma.inboundDocument.create({ + data: { ...base, sourceSystem: null, sourceId: null }, + }); + }); + + it("429s the token on its own bucket with Retry-After, and the owner's uploads still pass", async () => { + await seedUser({ inboundDocuments: true }); + token = await mintDocumentToken(); + process.env.DOCUMENT_UPLOAD_LIMIT_PER_HOUR = "1"; + + asToken(); + expect((await post(upload(pdf("one")))).status).toBe(201); + const limited = await post(upload(pdf("two"))); + expect(limited.status).toBe(429); + expect(Number(limited.headers.get("retry-after"))).toBeGreaterThan(0); + expect((await limited.json()).meta.errorCode).toBe( + "documents.inbound.rateLimited", + ); + + asCookie(); + const own = await post(upload(pdf("three"))); + expect(own.status).toBe(201); + // The cookie caller still gets the full row. + expect((await own.json()).data.title).toBeNull(); + }); + + it("413s past the quota with the figures, storing nothing", async () => { + await seedUser({ inboundDocuments: true }); + token = await mintDocumentToken(); + await getPrismaClient().user.update({ + where: { id: userId }, + data: { documentQuotaBytes: BigInt(10) }, + }); + asToken(); + const res = await post(upload(PNG_1X1, {}, "x.png")); + expect(res.status).toBe(413); + const body = await res.json(); + expect(body.meta).toMatchObject({ + reason: "quotaExceeded", + quotaBytes: 10, + }); + expect( + await getPrismaClient().inboundDocument.count({ where: { userId } }), + ).toBe(0); + }); + + it("403 module.disabled while the vault is off", async () => { + await seedUser({ inboundDocuments: false }); + token = await mintDocumentToken(); + asToken(); + const res = await post(upload(pdf("x"))); + expect(res.status).toBe(403); + expect((await res.json()).meta.errorCode).toBe("module.disabled"); + }); +}); + +describe("a documents:write token reaches nothing else", () => { + it("is refused by every other vault leg, the mints and the measurement write", async () => { + await seedUser({ inboundDocuments: true }); + token = await mintDocumentToken(); + asToken(); + const created = await post(upload(pdf("seed"))); + expect(created.status).toBe(201); + const id = (await created.json()).data.id as string; + + const auth = { authorization: `Bearer ${token}` }; + const json = (body: unknown) => ({ + headers: { ...auth, "content-type": "application/json" }, + body: JSON.stringify(body), + }); + const u = (path: string) => `http://localhost${path}`; + type Call = [string, () => Promise]; + type Mod = Record; + const withId = async ( + route: Mod, + mod: string, + method: string, + init: RequestInit = { headers: auth }, + ): Promise => + (route[method] as (r: Request, c: Ctx) => Promise)( + new NextRequest(u(`/api/documents/inbound/${id}${mod}`), { + method, + ...init, + } as never), + ctx(id), + ); + const noId = async ( + route: Mod, + mod: string, + method: string, + init: RequestInit = { headers: auth }, + ): Promise => + (route[method] as (r: Request) => Promise)( + new NextRequest(u(`/api/documents/inbound${mod}`), { + method, + ...init, + } as never), + ); + const r = { + list: await import("@/app/api/documents/inbound/route"), + usage: await import("@/app/api/documents/inbound/usage/route"), + capability: await import("@/app/api/documents/inbound/capability/route"), + reindex: await import("@/app/api/documents/inbound/reindex/route"), + bulk: await import("@/app/api/documents/inbound/bulk/route"), + byId: await import("@/app/api/documents/inbound/[id]/route"), + original: await import("@/app/api/documents/inbound/[id]/original/route"), + thumbnail: + await import("@/app/api/documents/inbound/[id]/thumbnail/route"), + restore: await import("@/app/api/documents/inbound/[id]/restore/route"), + extract: await import("@/app/api/documents/inbound/[id]/extract/route"), + index: await import("@/app/api/documents/inbound/[id]/index/route"), + summary: await import("@/app/api/documents/inbound/[id]/summary/route"), + suggest: await import("@/app/api/documents/inbound/[id]/suggest/route"), + chat: await import("@/app/api/documents/inbound/[id]/chat/route"), + confirm: await import("@/app/api/documents/inbound/[id]/confirm/route"), + } as unknown as Record; + + const calls: Call[] = [ + ["GET list", () => noId(r.list, "", "GET")], + ["GET usage", () => noId(r.usage, "/usage", "GET")], + ["GET capability", () => noId(r.capability, "/capability", "GET")], + ["POST reindex", () => noId(r.reindex, "/reindex", "POST")], + [ + "POST bulk", + () => + noId(r.bulk, "/bulk", "POST", json({ action: "delete", ids: [id] })), + ], + ["GET detail", () => withId(r.byId, "", "GET")], + ["PATCH detail", () => withId(r.byId, "", "PATCH", json({ title: "x" }))], + ["DELETE detail", () => withId(r.byId, "", "DELETE")], + ["GET original", () => withId(r.original, "/original", "GET")], + ["GET thumbnail", () => withId(r.thumbnail, "/thumbnail", "GET")], + ["POST restore", () => withId(r.restore, "/restore", "POST")], + ["POST extract", () => withId(r.extract, "/extract", "POST")], + ["POST index", () => withId(r.index, "/index", "POST")], + ["POST summary", () => withId(r.summary, "/summary", "POST")], + ["POST suggest", () => withId(r.suggest, "/suggest", "POST")], + ["GET chat", () => withId(r.chat, "/chat", "GET")], + [ + "POST confirm", + () => withId(r.confirm, "/confirm", "POST", json({ decisions: [] })), + ], + [ + "POST documents mint", + async () => { + const { POST } = await import("@/app/api/tokens/documents/route"); + return POST( + new NextRequest(u("/api/tokens/documents"), { + method: "POST", + ...json({ name: "second" }), + } as never), + ); + }, + ], + [ + "POST measurements", + async () => { + const { POST } = await import("@/app/api/measurements/route"); + return (POST as unknown as (r: Request) => Promise)( + new NextRequest(u("/api/measurements"), { + method: "POST", + ...json({ type: "WEIGHT", value: 80, measuredAt: new Date() }), + } as never), + ); + }, + ], + ]; + + const outcomes: Record = {}; + for (const [name, call] of calls) { + outcomes[name] = (await call()).status; + } + // Every leg refuses: 403 on a scope decision, 401 on the cookie-only mint. + for (const [name, status] of Object.entries(outcomes)) { + expect({ name, refused: status === 401 || status === 403 }).toEqual({ + name, + refused: true, + }); + } + expect(outcomes["POST documents mint"]).toBe(401); + + // And none of them changed the document. + const row = await getPrismaClient().inboundDocument.findUniqueOrThrow({ + where: { id }, + omit: { contentEncrypted: true }, + }); + expect(row.deletedAt).toBeNull(); + expect(row.title).toBeNull(); + }); +});