diff --git a/.env.example b/.env.example index aa5a7e3c3..5474388c2 100644 --- a/.env.example +++ b/.env.example @@ -103,6 +103,20 @@ API_TOKEN_HMAC_KEY="REPLACE-WITH-OUTPUT-OF-openssl-rand-hex-32" # NOTIFICATION_PRIVATE_ORIGINS="" +# ----------------------------------------------------------------------------- +# Document picker: Paperless-ngx and Papra -- optional +# ----------------------------------------------------------------------------- +# The Paperless-ngx and Papra instances people may connect to pick documents +# from, comma-separated exact origins (scheme://host[:port], no path), e.g. +# "https://paperless.example.com,http://papra.lan:1221". Unset, the document +# picker is switched off. Only a listed origin is ever contacted, public or +# private; requests are pinned to it, redirects are not followed, and +# link-local, metadata and unspecified addresses stay refused even when +# listed. A malformed entry is logged once and grants nothing. +# See docs/self-hosting/document-sources.md. +# DOCUMENT_SOURCE_ORIGINS="" + + # ----------------------------------------------------------------------------- # AI Provider (optional) # ----------------------------------------------------------------------------- diff --git a/.env.production.example b/.env.production.example index ead149065..a297815e3 100644 --- a/.env.production.example +++ b/.env.production.example @@ -85,6 +85,20 @@ API_TOKEN_HMAC_KEY="" # NOTIFICATION_PRIVATE_ORIGINS="" +# ----------------------------------------------------------------------------- +# Document picker: Paperless-ngx and Papra -- optional +# ----------------------------------------------------------------------------- +# The Paperless-ngx and Papra instances people may connect to pick documents +# from, comma-separated exact origins (scheme://host[:port], no path), e.g. +# "https://paperless.example.com,http://papra.lan:1221". Unset, the document +# picker is switched off. Only a listed origin is ever contacted, public or +# private; requests are pinned to it, redirects are not followed, and +# link-local, metadata and unspecified addresses stay refused even when +# listed. A malformed entry is logged once and grants nothing. +# See docs/self-hosting/document-sources.md. +# DOCUMENT_SOURCE_ORIGINS="" + + # ----------------------------------------------------------------------------- # WHOOP -- optional integration # ----------------------------------------------------------------------------- diff --git a/docker-compose.yml b/docker-compose.yml index a24b90ba0..c595de613 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -215,6 +215,13 @@ x-healthlog-env: &healthlog_env # and unspecified addresses stay refused even when listed. See # docs/self-hosting/notifications.md. NOTIFICATION_PRIVATE_ORIGINS: "${NOTIFICATION_PRIVATE_ORIGINS:-}" + # Paperless-ngx and Papra instances the document picker may reach, + # comma-separated exact origins (e.g. "https://paperless.example.com, + # http://papra.lan:1221"). Unset, the picker is off. Only listed origins are + # ever contacted, public or private; redirects are not followed and + # link-local, metadata and unspecified addresses stay refused even when + # listed. See docs/self-hosting/document-sources.md. + DOCUMENT_SOURCE_ORIGINS: "${DOCUMENT_SOURCE_ORIGINS:-}" # Open-Meteo endpoints for the environment/weather correlation. Default to # the hosted service; point them at your own Open-Meteo deployment to keep # the coordinate lookups off a third party. diff --git a/docs/api/openapi.yaml b/docs/api/openapi.yaml index 2dc9a4a6f..c19062979 100644 --- a/docs/api/openapi.yaml +++ b/docs/api/openapi.yaml @@ -15722,21 +15722,23 @@ paths: **Importing from another system (v1.39.2).** Optional `sourceSystem` (`PAPERLESS`, `PAPRA`, `OTHER`) and - `sourceId` (its id there, printable, up to 128 characters; needs `sourceSystem`) key the upload. A re-send with - a known key is a duplicate: 200 with the live row (`meta.duplicate: true`), or, when the owner DELETED that - document, 200 with `data.deleted: true` and `meta.deleted: true` and nothing stored; this holds after the 30-day - purge too. The key may also ride the query string (`?sourceSystem=…&sourceId=…`): a key the vault already holds - is then answered at once, the file is not stored or processed, and the request counts only toward the lookup - allowance shared with `GET /api/documents/inbound/source` (5000 an hour per token, per person for a session; 429 - past it), not toward the hourly upload limit. A key in both places must agree (else 422). Every other upload - counts toward the hourly upload limit, duplicates included. A key that is answered with an existing document by - its bytes is remembered for that document, so deleting it keeps that key deleted too; one document takes at most - 20 such keys, beyond which the upload is refused 409 (`documents.inbound.sourceAliasLimit`). `OTHER` is one - shared namespace per account: two different systems both sent as `OTHER` must not reuse each other's ids. - `aiRead=defer` holds back automatic AI reading for this upload: the thumbnail and a local text index still run, - the summary and lab staging do not, and the document stays out of every automatic AI path (the summary catch-up, - the automatic index, Index all for search) until the person reads it with AI or generates its summary. Absent, - behaviour is unchanged. + `sourceId` (its id there, printable, up to 128 characters, never `.` or `..`; needs `sourceSystem`) key the + upload, optionally with `sourceInstance` (v1.39.3: the address of the Paperless-ngx or Papra it came from, + normalised to its origin; without it the key matches any instance of the system). A re-send with a known key is + a duplicate: 200 with the live row (`meta.duplicate: true`), or, when the owner DELETED that document, 200 with + `data.deleted: true` and `meta.deleted: true` and nothing stored; this holds after the 30-day purge too. The key + may also ride the query string (`?sourceSystem=…&sourceId=…`): a key the vault already holds is then answered at + once, the file is not stored or processed, and the request counts only toward the lookup allowance shared with + `GET /api/documents/inbound/source` (5000 an hour per token, per person for a session; 429 past it), not toward + the hourly upload limit. A key in both places must agree (else 422). Every other upload counts toward the hourly + upload limit, duplicates included. A key that is answered with an existing document by its bytes is remembered + for that document, so deleting it keeps that key deleted too; one document takes at most 20 such keys, beyond + which the upload is refused 409 (`documents.inbound.sourceAliasLimit`). `OTHER` is one shared namespace per + account: two different systems both sent as `OTHER` must not reuse each other's ids. `aiRead=defer` holds back + automatic AI reading for this upload: the thumbnail and a local text index still run, the summary and lab + staging do not, and the document stays out of every automatic AI path (the summary catch-up, the automatic + index, Index all for search) until the person reads it with AI or generates its summary. Absent, behaviour is + unchanged. **Narrow token.** Also reachable with a `documents:write` Bearer (minted at `POST /api/tokens/documents`), which @@ -15768,6 +15770,16 @@ paths: type: string maxLength: 128 description: Import source key, second half. Both halves or neither; half a key is 422. + - name: sourceInstance + in: query + required: false + schema: + type: string + maxLength: 512 + description: "Optional, with a key. The instance of `sourceSystem` the id belongs to, as its http(s) address; normalised + to the origin (`scheme://host[:port]`). The same id in two instances is two documents. Optional: a key + stored without an instance (every upload before v1.39.3) matches any instance of its system, and a key sent + without one matches any stored instance. If also sent in the form it must agree." requestBody: required: true content: @@ -15822,6 +15834,13 @@ paths: documents. type: string maxLength: 128 + sourceInstance: + description: "Optional; requires `sourceId`. The instance of `sourceSystem` the id belongs to, as its http(s) address; + normalised to the origin (`scheme://host[:port]`). The same id in two instances is two documents. + Optional: a key stored without an instance (every upload before v1.39.3) matches any instance of its + system, and a key sent without one matches any stored instance." + type: string + maxLength: 512 aiRead: description: Optional. `defer` holds back automatic AI reading for this upload (local index and thumbnail only). type: string @@ -15897,6 +15916,16 @@ paths: schema: type: string maxLength: 128 + - name: sourceInstance + in: query + required: false + schema: + type: string + maxLength: 512 + description: "The instance of `sourceSystem` the id belongs to, as its http(s) address; normalised to the origin + (`scheme://host[:port]`). The same id in two instances is two documents. Optional: a key stored without an + instance (every upload before v1.39.3) matches any instance of its system, and a key sent without one + matches any stored instance." responses: "200": description: The answer for this key. @@ -20967,6 +20996,377 @@ paths: application/json: schema: $ref: "#/components/schemas/ErrorEnvelope" + /api/documents/sources: + get: + tags: + - Documents + summary: "Document archives: availability and connections" + description: "Whether the operator enabled the document picker (`available`, true when `DOCUMENT_SOURCE_ORIGINS` lists + at least one origin), the listed origins, and the caller's own saved connections to Paperless-ngx or Papra. A + connection never carries its token: `hasToken` is always true for a saved one, and no route returns the token in + whole or in part. `originAllowed` is false once the operator removed the connection's origin from the list. No + request leaves the server. Needs neither the Documents module nor the list: with the list unset, `available` is + false and any connection still stored is listed (with `originAllowed: false`) so it can be removed. Cookie + session only; not delegable." + security: &a52 + - cookieAuth: [] + responses: + "200": + description: The picker's state for the caller. + content: + application/json: + schema: + $ref: "#/components/schemas/DocumentSourcesStatusEnvelope" + "401": *a1 + "403": &a53 + description: "Refused before any request to the source: `documents.sources.browserOnly` for a Bearer caller, + `sharing.not_permitted` while acting on another person's record (a delegate or guardian never uses the + owner's archive), or `module.disabled` when the Documents module is off." + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorEnvelope" + "422": *a4 + "429": *a5 + /api/documents/sources/{system}: + put: + tags: + - Documents + summary: Save a connection to a document archive + description: "Saves or replaces the caller's connection to Paperless-ngx (`paperless`: base address and API token) or + Papra (`papra`: base address, organization id, API key with `documents:read` and `tags:read`). The base address + may carry a path; its origin must be listed in `DOCUMENT_SOURCE_ORIGINS`. The connection is tested live before + anything is stored. Leaving out `token` keeps the stored one, but only when the new address is on the same + origin as the saved one: the first save and a move to another origin must carry it (422 + `documents.sources.tokenRequired`), so a stored token is never sent to an address it was not saved for. A Papra + organization id is letters, digits, `_` and `-`. The token is stored encrypted (AES-256-GCM) and never returned. + Ten saves and tests a minute per person. Cookie session only; not delegable." + security: *a52 + parameters: + - &a54 + name: system + in: path + required: true + description: "`paperless` (Paperless-ngx) or `papra` (Papra)." + schema: + type: string + enum: + - paperless + - papra + requestBody: + required: true + content: + application/json: + schema: + type: object + properties: + baseUrl: + type: string + minLength: 1 + maxLength: 2048 + organizationId: + type: string + pattern: ^org_[a-z0-9]{24}$ + token: + type: string + minLength: 1 + maxLength: 512 + pattern: ^[\x21-\x7e]+$ + required: + - baseUrl + responses: + "200": + description: Saved and verified. The connection, without its token. + content: + application/json: + schema: + $ref: "#/components/schemas/DocumentSourceConnectionEnvelope" + "401": *a1 + "403": *a53 + "404": &a55 + description: "`documents.sources.unavailable`: the operator has not listed any origin in `DOCUMENT_SOURCE_ORIGINS`, so + the picker is off. `documents.sources.notConnected`: no connection is saved for this system. + `documents.sources.notFound`: the source has no such document, or (Papra) no such organization, including + one the key's user is not a member of (Papra 403 `user.not_in_organization`) and one Papra cannot parse + (400)." + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorEnvelope" + "422": + description: "`documents.sources.originNotAllowed`, `documents.sources.invalidAddress`, + `documents.sources.organizationRequired` (Papra), `documents.sources.tokenRequired` (first save without a + token), or body validation." + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorEnvelope" + "429": *a5 + "502": &a56 + description: "The source could not answer usefully, with the reason in `meta.errorCode` and, when the source answered at + all, its HTTP status in `meta.upstreamStatus` (its body is never passed on): `documents.sources.unreachable` + (no connection, timeout, or a destination refused at dial time), `documents.sources.redirected` (the source + answered with a redirect, which is never followed), `documents.sources.authRefused` (401/403 from the + source), `documents.sources.permissionMissing` (Papra: the key cannot read tags; needs `tags:read`), + `documents.sources.versionTooOld` (Paperless-ngx older than 2.16 / API version 9), + `documents.sources.badResponse` (not the documented JSON, or larger than 1 MiB)." + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorEnvelope" + delete: + tags: + - Documents + summary: Remove a connection to a document archive + description: Deletes the caller's connection for this system, and its token with it. Works even after the operator + removed the origin from the list, and with the Documents module switched off. `disconnected` is false when there + was nothing to delete. Cookie session only; not delegable. + security: *a52 + parameters: + - *a54 + responses: + "200": + description: Removed, or there was nothing to remove. + content: + application/json: + schema: + $ref: "#/components/schemas/DocumentSourceDisconnectEnvelope" + "401": *a1 + "403": *a53 + "422": *a4 + "429": *a5 + /api/documents/sources/{system}/test: + post: + tags: + - Documents + summary: Test a saved connection + description: One request to the source with the stored token. Answers `ok` with the round-trip time and stamps + `lastVerifiedAt`, or the reason in `meta.errorCode`. Shares the save's allowance of ten a minute. Cookie session + only; not delegable. + security: *a52 + parameters: + - *a54 + responses: + "200": + description: The source answered. + content: + application/json: + schema: + $ref: "#/components/schemas/DocumentSourceTestEnvelope" + "401": *a1 + "403": *a53 + "404": *a55 + "422": &a57 + description: "`documents.sources.originNotAllowed`: the base address's origin is not on the operator's list (checked on + every request, so a connection saved earlier is cut off once its origin is removed), or another validation + failure." + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorEnvelope" + "429": *a5 + "502": *a56 + /api/documents/sources/{system}/tags: + get: + tags: + - Documents + summary: Tags of a connected archive + description: The source's tags for the picker's filter, the first 250 by name, in the source's own ids. Shares the + search allowance of sixty a minute. Cookie session only; not delegable. + security: *a52 + parameters: + - *a54 + responses: + "200": + description: The tags. + content: + application/json: + schema: + $ref: "#/components/schemas/DocumentSourceTagsEnvelope" + "401": *a1 + "403": *a53 + "404": *a55 + "422": *a57 + "429": *a5 + "502": *a56 + /api/documents/sources/{system}/search: + get: + tags: + - Documents + summary: Search a connected archive by name, tag and date + description: "Searches the source 25 documents at a time. `q` matches the document's name (Paperless-ngx title, Papra + name); `tag` narrows to one of the source's tags by id; `from` / `to` to a date range, inclusive (Paperless-ngx + filters on `created__gte` / `created__lte`; Papra's range is applied to each page after it arrives). When a + filter applied after arrival empties a page, the route reads on, up to five source pages, and `page` in the + answer is the last source page read; ask for `page + 1` next. Each result carries what the vault already holds + under that source key: `new`, `imported` with the vault document's `documentId`, or `deleted` (deleted in + HealthLog; importing it again stores nothing). Nothing is stored. Sixty searches a minute, shared with the tag + read. Cookie session only; not delegable." + security: *a52 + parameters: + - *a54 + - name: q + in: query + required: false + schema: + type: string + maxLength: 200 + - name: tag + in: query + required: false + schema: + type: string + maxLength: 64 + - name: from + in: query + required: false + schema: + type: string + format: date + - name: to + in: query + required: false + schema: + type: string + format: date + - name: page + in: query + required: false + schema: + type: integer + minimum: 1 + maximum: 500 + default: 1 + responses: + "200": + description: One page of results. + content: + application/json: + schema: + $ref: "#/components/schemas/DocumentSourceSearchEnvelope" + "401": *a1 + "403": *a53 + "404": *a55 + "422": *a57 + "429": *a5 + "502": *a56 + /api/documents/sources/{system}/import: + post: + tags: + - Documents + summary: Import one picked document + description: "`sourceId` must be an id the system issues (Paperless-ngx: digits; Papra: letters, digits, `_`, `-`), else + 422 before anything is fetched. Fetches one document from the source and stores it in the vault through the same + path as an upload, with `sourceSystem` / `sourceId` / `sourceInstance` (the connection's origin) provenance; the + same id from another instance is another document. One import per key runs at a time; a concurrent second one + waits and then answers `duplicate`. The key check counts toward the lookup allowance, the fetch toward the + search allowance, and only a file that arrived toward the upload allowance. A source key the vault already holds + is answered without a download: `duplicate` for a live document (same id), `deleted` for one the person deleted + in HealthLog, which is never stored again. Bytes already in the vault under another key are `duplicate` too, and + the key is remembered for that document. `link` files the document against the record the picker was opened from + (a condition, a visit or a vaccination, which must be the caller's own and live; checked before anything is + fetched), including a duplicate. `kind` sets the document type (default OTHER). Automatic AI reading follows the + person's own setting. Counts against the person's upload allowance (60 an hour) and storage quota. 201 for a new + document, 200 otherwise. Cookie session only; not delegable." + security: *a52 + parameters: + - *a54 + requestBody: + required: true + content: + application/json: + schema: + type: object + properties: + sourceId: + type: string + minLength: 1 + maxLength: 128 + pattern: ^[\x21-\x7e]+$ + kind: + type: string + enum: + - DOCTOR_REPORT + - DISCHARGE_LETTER + - LAB_RESULT + - IMAGING + - PRESCRIPTION + - REFERRAL + - INSURANCE + - VACCINATION + - OTHER + link: + type: object + properties: + kind: + type: string + enum: + - conditionEpisode + - encounter + - vaccination + id: + type: string + minLength: 1 + maxLength: 64 + required: + - kind + - id + required: + - sourceId + responses: + "200": + description: Already held (`duplicate`) or deleted in HealthLog (`deleted`). + content: + application/json: + schema: + $ref: "#/components/schemas/DocumentSourceImportHeldEnvelope" + "201": + description: Imported. + content: + application/json: + schema: + $ref: "#/components/schemas/DocumentSourceImportEnvelope" + "401": *a1 + "403": *a53 + "404": + description: "`documents.sources.unavailable`: the operator has not listed any origin in `DOCUMENT_SOURCE_ORIGINS`, so + the picker is off. `documents.sources.notConnected`: no connection is saved for this system. + `documents.sources.notFound`: the source has no such document, or (Papra) no such organization, including + one the key's user is not a member of (Papra 403 `user.not_in_organization`) and one Papra cannot parse + (400). `documents.sources.linkTargetNotFound`: the record in `link` is not the caller's own live + conditionEpisode / encounter / vaccination." + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorEnvelope" + "409": + description: "`documents.inbound.sourceAliasLimit`: these bytes are already stored under too many source ids." + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorEnvelope" + "413": + description: "`documents.inbound.fileTooLarge` (with `maxFileBytes`) or `documents.inbound.quotaExceeded` (with + `quotaBytes` and `usedBytes`)." + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorEnvelope" + "415": + description: "`documents.inbound.fileType`: the file is not a type the vault accepts." + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorEnvelope" + "422": *a57 + "429": + description: "`documents.inbound.rateLimited` (the person's hourly upload allowance, with the rate-limit headers) or + `documents.inbound.uploadBusy` (too many uploads in progress; `Retry-After: 1`)." + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorEnvelope" + "502": *a56 /api/bugreport: post: security: [] @@ -30840,12 +31240,17 @@ components: documents.inbound.notIndexed, documents.inbound.pdfNeedsAnthropic, documents.inbound.providerUnsupported, documents.inbound.quotaExceeded, documents.inbound.rateLimited, documents.inbound.restoreDuplicate, documents.inbound.restoreGone, documents.inbound.sourceAliasLimit, - documents.inbound.uploadBusy, documents.inbound.uploadTimeout, documents.inbound.vaccinationNotFound; - encounter — encounter.invalid, encounter.link.invalid, encounter.practitioner-not-found, - encounter.reminder-conflict, encounter.reminder-not-found; environment — environment.invalid, - environment.no_home, environment.range_too_large, environment.travel.not_found; export — - export.selection.empty, export.selection.unknown_leaf; familyHistory — familyHistory.invalid; feedback — - feedback.body.invalid; health_score_config — health_score_config.invalid, + documents.inbound.uploadBusy, documents.inbound.uploadTimeout, documents.inbound.vaccinationNotFound, + documents.sources.authRefused, documents.sources.badResponse, documents.sources.browserOnly, + documents.sources.invalidAddress, documents.sources.linkTargetNotFound, documents.sources.notConnected, + documents.sources.notFound, documents.sources.organizationRequired, documents.sources.originNotAllowed, + documents.sources.permissionMissing, documents.sources.rateLimited, documents.sources.redirected, + documents.sources.tokenRequired, documents.sources.unavailable, documents.sources.unreachable, + documents.sources.versionTooOld; encounter — encounter.invalid, encounter.link.invalid, + encounter.practitioner-not-found, encounter.reminder-conflict, encounter.reminder-not-found; environment + — environment.invalid, environment.no_home, environment.range_too_large, environment.travel.not_found; + export — export.selection.empty, export.selection.unknown_leaf; familyHistory — familyHistory.invalid; + feedback — feedback.body.invalid; health_score_config — health_score_config.invalid, health_score_config.too_narrow; illness — illness.day-log.invalid, illness.disabled, illness.episode.chronic-no-resolve, illness.episode.invalid, illness.episode.invalid-window, illness.episode.parent-not-found, illness.insights.invalid; import — import.write_failed; ingest — @@ -35406,6 +35811,11 @@ components: - type: string maxLength: 128 - type: "null" + sourceInstance: + anyOf: + - type: string + maxLength: 512 + - type: "null" aiReadDeferred: type: boolean summary: @@ -72409,6 +72819,315 @@ components: - data - error additionalProperties: false + DocumentSourcesStatusEnvelope: + type: object + properties: + data: + $ref: "#/components/schemas/DocumentSourcesStatus" + error: + type: "null" + meta: + type: object + properties: + requestId: + type: string + additionalProperties: false + required: + - data + - error + additionalProperties: false + DocumentSourcesStatus: + type: object + properties: + available: + type: boolean + allowedOrigins: + type: array + items: + type: string + connections: + type: array + items: + $ref: "#/components/schemas/DocumentSourceConnection" + required: + - available + - allowedOrigins + - connections + additionalProperties: false + DocumentSourceConnection: + type: object + properties: + system: + type: string + enum: + - PAPERLESS + - PAPRA + baseUrl: + type: string + organizationId: + anyOf: + - type: string + - type: "null" + hasToken: + type: boolean + const: true + lastVerifiedAt: + anyOf: + - type: string + - type: "null" + originAllowed: + type: boolean + required: + - system + - baseUrl + - organizationId + - hasToken + - lastVerifiedAt + - originAllowed + additionalProperties: false + DocumentSourceConnectionEnvelope: + type: object + properties: + data: + $ref: "#/components/schemas/DocumentSourceConnection" + error: + type: "null" + meta: + type: object + properties: + requestId: + type: string + additionalProperties: false + required: + - data + - error + additionalProperties: false + DocumentSourceDisconnectEnvelope: + type: object + properties: + data: + $ref: "#/components/schemas/DocumentSourceDisconnect" + error: + type: "null" + meta: + type: object + properties: + requestId: + type: string + additionalProperties: false + required: + - data + - error + additionalProperties: false + DocumentSourceDisconnect: + type: object + properties: + disconnected: + type: boolean + required: + - disconnected + additionalProperties: false + DocumentSourceTestEnvelope: + type: object + properties: + data: + $ref: "#/components/schemas/DocumentSourceTest" + error: + type: "null" + meta: + type: object + properties: + requestId: + type: string + additionalProperties: false + required: + - data + - error + additionalProperties: false + DocumentSourceTest: + type: object + properties: + ok: + type: boolean + const: true + latencyMs: + type: integer + minimum: -9007199254740991 + maximum: 9007199254740991 + lastVerifiedAt: + type: string + required: + - ok + - latencyMs + - lastVerifiedAt + additionalProperties: false + DocumentSourceTagsEnvelope: + type: object + properties: + data: + $ref: "#/components/schemas/DocumentSourceTags" + error: + type: "null" + meta: + type: object + properties: + requestId: + type: string + additionalProperties: false + required: + - data + - error + additionalProperties: false + DocumentSourceTags: + type: object + properties: + tags: + type: array + items: + $ref: "#/components/schemas/DocumentSourceTag" + required: + - tags + additionalProperties: false + DocumentSourceTag: + type: object + properties: + id: + type: string + name: + type: string + required: + - id + - name + additionalProperties: false + DocumentSourceSearchEnvelope: + type: object + properties: + data: + $ref: "#/components/schemas/DocumentSourceSearch" + error: + type: "null" + meta: + type: object + properties: + requestId: + type: string + additionalProperties: false + required: + - data + - error + additionalProperties: false + DocumentSourceSearch: + type: object + properties: + results: + type: array + items: + $ref: "#/components/schemas/DocumentSourceResult" + page: + type: integer + minimum: -9007199254740991 + maximum: 9007199254740991 + hasMore: + type: boolean + required: + - results + - page + - hasMore + additionalProperties: false + DocumentSourceResult: + type: object + properties: + sourceId: + type: string + title: + type: string + date: + anyOf: + - type: string + - type: "null" + tags: + type: array + items: + type: string + sizeBytes: + anyOf: + - type: integer + minimum: -9007199254740991 + maximum: 9007199254740991 + - type: "null" + state: + type: string + enum: + - new + - imported + - deleted + documentId: + anyOf: + - type: string + - type: "null" + required: + - sourceId + - title + - date + - tags + - sizeBytes + - state + - documentId + additionalProperties: false + DocumentSourceImportHeldEnvelope: + type: object + properties: + data: + $ref: "#/components/schemas/DocumentSourceImport" + error: + type: "null" + meta: + type: object + properties: + requestId: + type: string + additionalProperties: false + required: + - data + - error + additionalProperties: false + DocumentSourceImport: + type: object + properties: + outcome: + type: string + enum: + - imported + - duplicate + - deleted + documentId: + anyOf: + - type: string + - type: "null" + linked: + type: boolean + required: + - outcome + - documentId + - linked + additionalProperties: false + DocumentSourceImportEnvelope: + type: object + properties: + data: + $ref: "#/components/schemas/DocumentSourceImport" + error: + type: "null" + meta: + type: object + properties: + requestId: + type: string + additionalProperties: false + required: + - data + - error + additionalProperties: false __schema0: anyOf: - type: string diff --git a/docs/integrations/document-import.md b/docs/integrations/document-import.md index 48e7d39c1..d7a0163f1 100644 --- a/docs/integrations/document-import.md +++ b/docs/integrations/document-import.md @@ -43,12 +43,13 @@ This needs Paperless-ngx 3.0 or later, the first release with the - **Use parameters for webhook body:** on. **Send webhook payload as JSON:** off. **Webhook params:** - | Name | Value | - | -------------- | ---------------------------------------------------- | - | `title` | `{{doc_title}}` | - | `documentDate` | `{{created_year}}-{{created_month}}-{{created_day}}` | - | `sourceSystem` | `PAPERLESS` | - | `sourceId` | `{{doc_id}}` | + | Name | Value | + | ---------------- | -------------------------------------------------------- | + | `title` | `{{doc_title}}` | + | `documentDate` | `{{created_year}}-{{created_month}}-{{created_day}}` | + | `sourceSystem` | `PAPERLESS` | + | `sourceId` | `{{doc_id}}` | + | `sourceInstance` | `https://paperless.example.com` (your Paperless address) | - **Include document:** on. Paperless then sends the original file along with the parameters, which is exactly what HealthLog expects. @@ -151,8 +152,15 @@ What it does: quick. Documents you deleted in HealthLog stay deleted, also after they are removed for good. The same goes for a file you had already uploaded by hand: the import recognises it by its content (sending it once to - compare), and if you delete it later, the import leaves it deleted. An - interrupted run simply picks up where it stopped. + compare), and if you delete it later, the import leaves it deleted. Since + v1.39.3 a deleted document is also recognised by its content while it is + in the 30-day undo window: the same file arriving from another system or + under a new id is answered "deleted" and not stored again. After the purge + HealthLog keeps the ids it saw, not the content. An interrupted run simply + picks up where it stopped. +- Once HealthLog has refused one file as too large, the script skips any + further document the source already lists as larger (Papra lists sizes) + without downloading it. - When HealthLog asks it to slow down, it waits as long as HealthLog says and carries on. By default a document token may send 120 uploads an hour, so an archive of a thousand documents takes a night. Every upload that @@ -231,6 +239,15 @@ per account: if you push documents from two other systems, make sure their ids cannot collide, for example by prefixing them (`nextcloud-123`, `scanner-123`). +Since v1.39.3 a key can also name its instance: `sourceInstance`, the +address of the Paperless-ngx or Papra it came from (only its origin, +`scheme://host[:port]`, counts). With it, document 123 on one Paperless and +document 123 on another are two documents. The import script sends it +automatically; add it to a Paperless workflow as shown above. A key sent or +stored without an instance (everything imported with v1.39.2, or a workflow +without the parameter) matches that id on any instance of its system, so +nothing imported earlier is stored twice or brought back after a deletion. + The same file sent under a different id is stored once, and HealthLog remembers the extra id for it, up to 20 extra ids per document. A 21st is refused with `409` (`documents.inbound.sourceAliasLimit`) and nothing is diff --git a/docs/self-hosting/document-sources.md b/docs/self-hosting/document-sources.md new file mode 100644 index 000000000..4a30cdd2c --- /dev/null +++ b/docs/self-hosting/document-sources.md @@ -0,0 +1,101 @@ +# Document picker — Paperless-ngx and Papra + +People who keep their paperwork in Paperless-ngx or Papra can search it from +HealthLog and import the documents they pick: under "Link a document" on a +visit, a vaccination or a condition, and from the Documents page. They search +by document name, narrow by tag and date, tick what they want, and HealthLog +fetches those files and stores its own encrypted copy, exactly as if they had +been uploaded. + +The picker is off until the operator names the instances HealthLog may +contact. + +## Turning it on + +List the origins in the server environment: + +```env +DOCUMENT_SOURCE_ORIGINS="https://paperless.example.com,http://papra.lan:1221" +``` + +The variable is on the `docker-compose.yml` whitelist, so setting it in `.env` +is enough; restart the app container afterwards. Unset or empty, the settings +card and every import button stay hidden and the routes answer 404. + +Each entry is one `scheme://host[:port]`, the grammar +`NOTIFICATION_PRIVATE_ORIGINS` uses (see [notifications](notifications.md)): + +- only `http` and `https`; scheme, host and port must match what a person + enters (`https://paperless.example.com:8443` and + `https://paperless.example.com` are two different entries); +- no path, query, credentials, wildcard or CIDR; a person may still enter a + base address with a path (`https://example.com/paperless`) as long as its + origin is listed; +- the unspecified address, link-local and the cloud-metadata range can never + be listed, and a listed name that resolves there is refused when HealthLog + connects; +- a malformed entry is logged once at startup, reduced to scheme and host, and + grants nothing; the valid entries beside it keep working. + +Unlike the notification list, this list is the whole decision: an origin +that is not listed is refused even when it is a public address. Removing an +entry cuts off the connections that point at it on their next request; people +can still delete their stored connection. + +## What each person sets up + +Settings → Integrations → Document archives: + +- **Paperless-ngx:** the base address and an API token (Paperless-ngx: My + Profile → API token). Paperless-ngx 2.16 or later (API version 9). +- **Papra:** the base address, the organization id (from the Papra address + bar, `org_` followed by 24 letters and digits) and an API key with + `documents:read` and `tags:read`. The connection test reads the tags too, + so a key without `tags:read` is refused at save. + +HealthLog tests the connection before saving it. A saved token is only ever +sent to the address it was saved for: changing the address to another origin +asks for the token again. The token is stored +encrypted (AES-256-GCM, rotated with the other keys by +`scripts/rotate-encryption-key.ts`) and is never shown again, not even in +part. + +## What leaves the server, and when + +Only requests the person starts: a search, the tag list for the filter, a +connection test, and one download per picked document. There is no background +sync and no polling. Every request goes to the listed origin only, with +redirects refused, a 10-second limit for searches and 60 seconds for a file, +and a size cap on every answer (1 MiB for a search page, the vault's per-file +limit for a document). + +## Limits and rules + +- Name search on Paperless-ngx uses its own title filter. On a Paperless-ngx + that stores its data in SQLite, that filter ignores upper and lower case + only for plain ASCII letters: searching `röntgen` does not find `Röntgen`. + That is Paperless-ngx's behaviour, not HealthLog's; PostgreSQL-backed + Paperless does not have it. +- A document deleted in HealthLog is recognised by its content during the + 30-day undo window, so the same file from another archive is not stored + again; after the purge only its ids are remembered. + +- Imports count against the person's normal upload allowance (60 an hour) and + storage quota. +- A document that is already in HealthLog is not stored twice, and one the + person deleted in HealthLog is not brought back by picking it again. Each + import is keyed by system, instance (the archive's origin) and document id, + so the same id on two Paperless-ngx instances is two documents; a key + imported before v1.39.3 carries no instance and matches any instance. These + are the same rules the document token and `scripts/import-documents.mjs` + follow. +- A picked document follows the person's own automatic AI reading setting. +- Only the owner of a record can connect an archive or import from it. People + with shared access to a record, whatever their level, do not see the picker. +- Connections are not part of backups. After a restore on another server, + each person reconnects (one form); the imported documents themselves are in + the backup with the rest of the vault. +- Delete all data and account deletion remove the connection and its token. +- A stored connection stays visible under Settings, read-only, when this + variable is unset or the person switched Documents off, so it can still be + removed. diff --git a/messages/de.json b/messages/de.json index 06801e37e..3dead10f6 100644 --- a/messages/de.json +++ b/messages/de.json @@ -6698,6 +6698,35 @@ "showChecklist": "Checkliste wieder anzeigen", "checklistShown": "Die Checkliste ist wieder auf dem Dashboard.", "checklistNothingHidden": "Die Checkliste war nicht ausgeblendet; sie öffnet sich auf dem Dashboard." + }, + "documentSources": { + "title": "Dokumentarchive", + "description": "Paperless-ngx oder Papra aus HealthLog durchsuchen und ausgewählte Dokumente importieren.", + "explainer": "Ein Import-Knopf erscheint überall, wo du Dokumente verknüpfst: bei Arztbesuchen, Impfungen und Erkrankungen sowie unter Dokumente. HealthLog speichert von jedem importierten Dokument eine eigene verschlüsselte Kopie und fragt dein Archiv nur, wenn du suchst oder importierst.", + "systemLabel": "Archiv", + "statusConnected": "Verbunden", + "statusNotConnected": "Nicht verbunden", + "connectedSince": "Verbunden, zuletzt geprüft am {date}", + "originRemoved": "Diese Adresse steht nicht mehr auf der Liste der erlaubten Archive dieses Servers. Frag die Administration oder speichere eine andere Adresse.", + "baseUrl": "Adresse", + "allowedOrigins": "Auf diesem Server erlaubt: {origins}", + "organizationId": "Organisations-ID", + "organizationHelp": "Der Teil nach /organizations/ in der Adresszeile von Papra. Papra-Organisations-IDs sehen aus wie org_ gefolgt von 24 Buchstaben und Ziffern.", + "token": "API-Token", + "tokenSaved": "Für diese Adresse gespeichert. Leer lassen, um es zu behalten.", + "tokenHelpPaperless": "In Paperless-ngx: Mein Profil, API-Token.", + "tokenHelpPapra": "In Papra: ein API-Schlüssel mit documents:read und tags:read.", + "save": "Speichern und testen", + "saving": "Wird getestet …", + "saved": "Verbunden. Das Archiv hat geantwortet.", + "test": "Testen", + "testOk": "Das Archiv hat in {latency} ms geantwortet.", + "disconnect": "Trennen", + "disconnectTitle": "{name} trennen?", + "disconnectDescription": "HealthLog vergisst Adresse und Token. Bereits importierte Dokumente bleiben in HealthLog.", + "unavailableExplainer": "Dieser Server erlaubt keine Dokumentarchive mehr, die Verbindung unten ist daher nicht nutzbar. Du kannst sie weiterhin entfernen.", + "moduleOffExplainer": "Dokumente sind ausgeschaltet, die Verbindung unten wird daher nicht genutzt. Du kannst sie weiterhin entfernen.", + "tokenNewAddress": "Für eine neue Adresse wird das API-Token erneut gebraucht." } }, "admin": { @@ -10147,6 +10176,70 @@ "body": "Beim Lesen mit KI geht dies an deinen KI-Dienst außerhalb dieses Servers. Darf HealthLog Dokumente, Laborbefunde und Medikamentenbeschreibungen zum Lesen senden?", "grant": "Lesen erlauben", "error": "Die Einwilligung konnte nicht gespeichert werden. Bitte versuch es erneut." + }, + "sourcePicker": { + "open": "Aus Archiv importieren", + "openNamed": "Aus {name} importieren", + "title": "Aus {name} importieren", + "description": "Nach Namen suchen, dann die Dokumente anhaken, die HealthLog übernehmen soll.", + "sourceLabel": "Archiv", + "searchLabel": "Dokumentname", + "searchPlaceholder": "Nach Namen suchen", + "tagLabel": "Tag", + "tagAny": "Alle Tags", + "fromLabel": "Von", + "toLabel": "Bis", + "clearFilters": "Tag und Datum zurücksetzen", + "rangeInvalid": "Das Enddatum liegt vor dem Startdatum.", + "empty": "{name} hat keine Dokumente zum Anzeigen.", + "noMatch": "Keine passenden Dokumente.", + "loadMore": "Mehr laden", + "stateImported": "In HealthLog", + "stateDeleted": "In HealthLog gelöscht", + "selectedCount": "{count} von höchstens {max} ausgewählt", + "importAction": "{count} importieren", + "importing": "Import läuft, ein Dokument nach dem anderen …", + "done": "Fertig", + "summary": "Importiert: {imported}. Schon vorhanden: {present}. Nicht importiert: {failed}.", + "stopped": "Abgebrochen: {reason}", + "result": { + "waiting": "Wartet", + "working": "Wird importiert …", + "imported": "Importiert", + "importedLinked": "Importiert und verknüpft", + "duplicate": "Schon in HealthLog", + "duplicateLinked": "Schon in HealthLog, jetzt verknüpft", + "deleted": "In HealthLog gelöscht, bleibt gelöscht" + }, + "errors": { + "unavailable": "Dokumentarchive sind auf diesem Server nicht freigeschaltet.", + "notConnected": "Dieses Archiv ist nicht verbunden. Verbinde es unter Einstellungen, Integrationen.", + "originNotAllowed": "Diese Adresse steht nicht auf der Liste der erlaubten Archive dieses Servers. Bitte die Administration, sie aufzunehmen.", + "unreachable": "Das Archiv war nicht erreichbar. Prüfe, ob es läuft und die Adresse stimmt.", + "redirected": "Das Archiv hat die Anfrage umgeleitet. Trage die Adresse ein, auf die es umleitet.", + "authRefused": "Das Archiv hat das Token abgelehnt. Prüfe das Token und seine Berechtigungen.", + "versionTooOld": "Das Archiv ist zu alt. Nötig ist Paperless-ngx 2.16 oder neuer.", + "badResponse": "Die Antwort des Archivs ließ sich nicht lesen. Prüfe, ob die Adresse auf das Archiv selbst zeigt.", + "notFound": "Das Archiv kennt dieses Dokument nicht, oder die Organisations-ID ist falsch oder nicht deine.", + "linkTargetNotFound": "Der Eintrag, mit dem verknüpft werden sollte, existiert nicht mehr.", + "rateLimited": "Zu viele Anfragen. Warte eine Minute und versuche es erneut.", + "browserOnly": "Dokumentarchive lassen sich nur im Browser verwenden.", + "invalidAddress": "Gib die vollständige Adresse ein, beginnend mit https:// oder http://.", + "organizationRequired": "Gib die Organisations-ID aus Papra ein.", + "tokenRequired": "Gib das API-Token ein. Eine neue Adresse braucht es erneut.", + "fileTooLarge": "Die Datei ist größer, als HealthLog annimmt.", + "quotaExceeded": "Dein Dokumentenspeicher ist voll.", + "fileType": "HealthLog nimmt diesen Dateityp nicht an.", + "uploadLimit": "Das stündliche Upload-Limit ist erreicht. Versuche es später erneut.", + "uploadBusy": "Andere Uploads laufen noch. Versuche es gleich noch einmal.", + "moduleDisabled": "Dokumente sind in den Einstellungen ausgeschaltet.", + "generic": "Etwas ist schiefgelaufen. Versuche es erneut.", + "permissionMissing": "Der Papra-Schlüssel darf keine Tags lesen. Gib ihm die Berechtigung tags:read." + }, + "pickMore": "Weitere auswählen", + "finishedHint": "Suche erneut oder ändere einen Filter, um weitere Dokumente auszuwählen.", + "maxReached": "Du kannst höchstens {max} Dokumente auf einmal importieren.", + "noMatchHint": "Versuche einen Teil des Namens oder setze Tag und Datum zurück." } }, "selfDescription": { diff --git a/messages/en.json b/messages/en.json index fa1fb5458..2e3c9feb5 100644 --- a/messages/en.json +++ b/messages/en.json @@ -6698,6 +6698,35 @@ "showChecklist": "Show the checklist again", "checklistShown": "The checklist is back on the dashboard.", "checklistNothingHidden": "The checklist was not hidden; it opens on the dashboard." + }, + "documentSources": { + "title": "Document archives", + "description": "Search Paperless-ngx or Papra from HealthLog and import the documents you pick.", + "explainer": "An import button appears wherever you link documents: on a visit, a vaccination and a condition, and in Documents. HealthLog keeps its own encrypted copy of each document you import, and asks your archive only when you search or import.", + "systemLabel": "Archive", + "statusConnected": "Connected", + "statusNotConnected": "Not connected", + "connectedSince": "Connected, last checked {date}", + "originRemoved": "This address is no longer on the server's list of allowed archives. Ask the administrator, or save another address.", + "baseUrl": "Address", + "allowedOrigins": "Allowed on this server: {origins}", + "organizationId": "Organization id", + "organizationHelp": "The part after /organizations/ in the Papra address bar. Papra organization ids look like org_ followed by 24 letters and digits.", + "token": "API token", + "tokenSaved": "Saved for this address. Leave empty to keep it.", + "tokenHelpPaperless": "In Paperless-ngx: My Profile, API token.", + "tokenHelpPapra": "In Papra: an API key with documents:read and tags:read.", + "save": "Save and test", + "saving": "Testing…", + "saved": "Connected. The archive answered.", + "test": "Test", + "testOk": "The archive answered in {latency} ms.", + "disconnect": "Disconnect", + "disconnectTitle": "Disconnect {name}?", + "disconnectDescription": "HealthLog forgets the address and the token. Documents you already imported stay in HealthLog.", + "unavailableExplainer": "This server no longer allows document archives, so the connection below cannot be used. You can still remove it.", + "moduleOffExplainer": "Documents are switched off, so the connection below is not used. You can still remove it.", + "tokenNewAddress": "A new address needs the API token again." } }, "admin": { @@ -10147,6 +10176,70 @@ "body": "Reading with AI sends this to your AI service outside this server. Allow HealthLog to send documents, lab reports and medication descriptions for reading?", "grant": "Allow reading", "error": "The consent could not be saved. Please try again." + }, + "sourcePicker": { + "open": "Import from archive", + "openNamed": "Import from {name}", + "title": "Import from {name}", + "description": "Search by name, then tick the documents HealthLog should copy.", + "sourceLabel": "Archive", + "searchLabel": "Document name", + "searchPlaceholder": "Search by name", + "tagLabel": "Tag", + "tagAny": "Any tag", + "fromLabel": "From", + "toLabel": "To", + "clearFilters": "Clear tag and dates", + "rangeInvalid": "The end date is before the start date.", + "empty": "{name} has no documents to show.", + "noMatch": "No documents match.", + "loadMore": "Load more", + "stateImported": "In HealthLog", + "stateDeleted": "Deleted in HealthLog", + "selectedCount": "{count} of up to {max} selected", + "importAction": "Import {count}", + "importing": "Importing, one document at a time…", + "done": "Done", + "summary": "Imported: {imported}. Already there: {present}. Not imported: {failed}.", + "stopped": "Stopped: {reason}", + "result": { + "waiting": "Waiting", + "working": "Importing…", + "imported": "Imported", + "importedLinked": "Imported and linked", + "duplicate": "Already in HealthLog", + "duplicateLinked": "Already in HealthLog, now linked", + "deleted": "Deleted in HealthLog, left alone" + }, + "errors": { + "unavailable": "Document archives are not enabled on this server.", + "notConnected": "This archive is not connected. Connect it under Settings, Integrations.", + "originNotAllowed": "This address is not on the server's list of allowed archives. Ask the administrator to add it.", + "unreachable": "The archive could not be reached. Check that it is running and that the address is right.", + "redirected": "The archive redirected the request. Enter the address it redirects to.", + "authRefused": "The archive refused the token. Check the token and its permissions.", + "versionTooOld": "The archive is too old. Paperless-ngx 2.16 or later is needed.", + "badResponse": "The archive's answer could not be read. Check that the address points at the archive itself.", + "notFound": "The archive has no such document, or the organization id is wrong or not yours.", + "linkTargetNotFound": "The record to link to no longer exists.", + "rateLimited": "Too many requests. Wait a minute and try again.", + "browserOnly": "Document archives can only be used in the browser.", + "invalidAddress": "Enter the full address, starting with https:// or http://.", + "organizationRequired": "Enter the Papra organization id.", + "tokenRequired": "Enter the API token. A new address needs it again.", + "fileTooLarge": "The file is larger than HealthLog accepts.", + "quotaExceeded": "Your document storage is full.", + "fileType": "HealthLog does not accept this file type.", + "uploadLimit": "The hourly upload limit is reached. Try again later.", + "uploadBusy": "Other uploads are still running. Try again in a moment.", + "moduleDisabled": "Documents are switched off in Settings.", + "generic": "Something went wrong. Try again.", + "permissionMissing": "The Papra key cannot read tags. Give it the tags:read permission." + }, + "pickMore": "Pick more", + "finishedHint": "Search again or change a filter to pick more documents.", + "maxReached": "You can import up to {max} documents at a time.", + "noMatchHint": "Try part of the name, or clear the tag and dates." } }, "selfDescription": { diff --git a/messages/es.json b/messages/es.json index 46f903102..5447d74ba 100644 --- a/messages/es.json +++ b/messages/es.json @@ -6698,6 +6698,35 @@ "showChecklist": "Mostrar la lista de nuevo", "checklistShown": "La lista vuelve a estar en el panel.", "checklistNothingHidden": "La lista no estaba oculta; se abre en el panel." + }, + "documentSources": { + "title": "Archivos de documentos", + "description": "Busca en Paperless-ngx o Papra desde HealthLog e importa los documentos que elijas.", + "explainer": "Aparece un botón de importar allí donde vinculas documentos: en visitas, vacunas y afecciones, y en Documentos. HealthLog guarda su propia copia cifrada de cada documento importado y solo consulta tu archivo cuando buscas o importas.", + "systemLabel": "Archivo", + "statusConnected": "Conectado", + "statusNotConnected": "No conectado", + "connectedSince": "Conectado, última comprobación el {date}", + "originRemoved": "Esta dirección ya no está en la lista de archivos permitidos del servidor. Consulta a la administración o guarda otra dirección.", + "baseUrl": "Dirección", + "allowedOrigins": "Permitido en este servidor: {origins}", + "organizationId": "Id de organización", + "organizationHelp": "La parte que sigue a /organizations/ en la barra de direcciones de Papra. Los id de organización de Papra tienen la forma org_ seguida de 24 letras y cifras.", + "token": "Token de la API", + "tokenSaved": "Guardado para esta dirección. Déjalo vacío para conservarlo.", + "tokenHelpPaperless": "En Paperless-ngx: Mi perfil, token de la API.", + "tokenHelpPapra": "En Papra: una clave de API con documents:read y tags:read.", + "save": "Guardar y probar", + "saving": "Probando…", + "saved": "Conectado. El archivo respondió.", + "test": "Probar", + "testOk": "El archivo respondió en {latency} ms.", + "disconnect": "Desconectar", + "disconnectTitle": "¿Desconectar {name}?", + "disconnectDescription": "HealthLog olvida la dirección y el token. Los documentos ya importados se quedan en HealthLog.", + "unavailableExplainer": "Este servidor ya no permite archivos de documentos, así que la conexión de abajo no se puede usar. Aún puedes eliminarla.", + "moduleOffExplainer": "Los documentos están desactivados, así que la conexión de abajo no se usa. Aún puedes eliminarla.", + "tokenNewAddress": "Una dirección nueva necesita de nuevo el token de la API." } }, "admin": { @@ -10147,6 +10176,70 @@ "body": "Leer con IA envía esto a tu servicio de IA fuera de este servidor. ¿Permitir que HealthLog envíe documentos, informes de laboratorio y descripciones de medicamentos para leerlos?", "grant": "Permitir lectura", "error": "No se pudo guardar el consentimiento. Inténtalo de nuevo." + }, + "sourcePicker": { + "open": "Importar desde archivo", + "openNamed": "Importar desde {name}", + "title": "Importar desde {name}", + "description": "Busca por nombre y marca los documentos que HealthLog debe copiar.", + "sourceLabel": "Archivo", + "searchLabel": "Nombre del documento", + "searchPlaceholder": "Buscar por nombre", + "tagLabel": "Etiqueta", + "tagAny": "Cualquier etiqueta", + "fromLabel": "Desde", + "toLabel": "Hasta", + "clearFilters": "Quitar etiqueta y fechas", + "rangeInvalid": "La fecha final es anterior a la inicial.", + "empty": "{name} no tiene documentos que mostrar.", + "noMatch": "Ningún documento coincide.", + "loadMore": "Cargar más", + "stateImported": "En HealthLog", + "stateDeleted": "Eliminado en HealthLog", + "selectedCount": "{count} de un máximo de {max} seleccionados", + "importAction": "Importar {count}", + "importing": "Importando, un documento tras otro…", + "done": "Hecho", + "summary": "Importados: {imported}. Ya estaban: {present}. No importados: {failed}.", + "stopped": "Detenido: {reason}", + "result": { + "waiting": "En espera", + "working": "Importando…", + "imported": "Importado", + "importedLinked": "Importado y vinculado", + "duplicate": "Ya está en HealthLog", + "duplicateLinked": "Ya está en HealthLog, ahora vinculado", + "deleted": "Eliminado en HealthLog, se deja así" + }, + "errors": { + "unavailable": "Los archivos de documentos no están habilitados en este servidor.", + "notConnected": "Este archivo no está conectado. Conéctalo en Ajustes, Integraciones.", + "originNotAllowed": "Esta dirección no está en la lista de archivos permitidos del servidor. Pide a la administración que la añada.", + "unreachable": "No se pudo acceder al archivo. Comprueba que está en marcha y que la dirección es correcta.", + "redirected": "El archivo redirigió la solicitud. Introduce la dirección a la que redirige.", + "authRefused": "El archivo rechazó el token. Comprueba el token y sus permisos.", + "versionTooOld": "El archivo es demasiado antiguo. Se necesita Paperless-ngx 2.16 o posterior.", + "badResponse": "No se pudo leer la respuesta del archivo. Comprueba que la dirección apunta al propio archivo.", + "notFound": "El archivo no tiene ese documento, o el id de organización es incorrecto o no es tuyo.", + "linkTargetNotFound": "El registro que se iba a vincular ya no existe.", + "rateLimited": "Demasiadas solicitudes. Espera un minuto y vuelve a intentarlo.", + "browserOnly": "Los archivos de documentos solo se pueden usar en el navegador.", + "invalidAddress": "Introduce la dirección completa, empezando por https:// o http://.", + "organizationRequired": "Introduce el id de organización de Papra.", + "tokenRequired": "Introduce el token de la API. Una dirección nueva lo necesita de nuevo.", + "fileTooLarge": "El archivo supera el tamaño que acepta HealthLog.", + "quotaExceeded": "Tu almacenamiento de documentos está lleno.", + "fileType": "HealthLog no acepta este tipo de archivo.", + "uploadLimit": "Se alcanzó el límite de subidas por hora. Inténtalo más tarde.", + "uploadBusy": "Aún hay otras subidas en curso. Inténtalo en un momento.", + "moduleDisabled": "Los documentos están desactivados en Ajustes.", + "generic": "Algo salió mal. Inténtalo de nuevo.", + "permissionMissing": "La clave de Papra no puede leer etiquetas. Dale el permiso tags:read." + }, + "pickMore": "Elegir más", + "finishedHint": "Busca de nuevo o cambia un filtro para elegir más documentos.", + "maxReached": "Puedes importar como máximo {max} documentos a la vez.", + "noMatchHint": "Prueba con parte del nombre o quita la etiqueta y las fechas." } }, "selfDescription": { diff --git a/messages/fr.json b/messages/fr.json index 722b77a24..e75cca526 100644 --- a/messages/fr.json +++ b/messages/fr.json @@ -6698,6 +6698,35 @@ "showChecklist": "Réafficher la liste de départ", "checklistShown": "La liste de départ est de retour sur le tableau de bord.", "checklistNothingHidden": "La liste n'était pas masquée ; elle s'ouvre sur le tableau de bord." + }, + "documentSources": { + "title": "Archives de documents", + "description": "Cherchez dans Paperless-ngx ou Papra depuis HealthLog et importez les documents choisis.", + "explainer": "Un bouton d'import apparaît partout où vous liez des documents : consultations, vaccinations, affections et Documents. HealthLog garde sa propre copie chiffrée de chaque document importé et n'interroge votre archive que lorsque vous cherchez ou importez.", + "systemLabel": "Archive", + "statusConnected": "Connecté", + "statusNotConnected": "Non connecté", + "connectedSince": "Connecté, dernière vérification le {date}", + "originRemoved": "Cette adresse ne figure plus dans la liste des archives autorisées du serveur. Voyez avec l'administration ou enregistrez une autre adresse.", + "baseUrl": "Adresse", + "allowedOrigins": "Autorisé sur ce serveur : {origins}", + "organizationId": "Identifiant d'organisation", + "organizationHelp": "La partie après /organizations/ dans la barre d'adresse de Papra. Les identifiants d'organisation Papra ont la forme org_ suivi de 24 lettres et chiffres.", + "token": "Jeton d'API", + "tokenSaved": "Enregistré pour cette adresse. Laissez vide pour le conserver.", + "tokenHelpPaperless": "Dans Paperless-ngx : Mon profil, jeton d'API.", + "tokenHelpPapra": "Dans Papra : une clé d'API avec documents:read et tags:read.", + "save": "Enregistrer et tester", + "saving": "Test en cours…", + "saved": "Connecté. L'archive a répondu.", + "test": "Tester", + "testOk": "L'archive a répondu en {latency} ms.", + "disconnect": "Déconnecter", + "disconnectTitle": "Déconnecter {name} ?", + "disconnectDescription": "HealthLog oublie l'adresse et le jeton. Les documents déjà importés restent dans HealthLog.", + "unavailableExplainer": "Ce serveur n'autorise plus les archives de documents ; la connexion ci-dessous ne peut donc pas servir. Vous pouvez toujours la supprimer.", + "moduleOffExplainer": "Les documents sont désactivés ; la connexion ci-dessous n'est donc pas utilisée. Vous pouvez toujours la supprimer.", + "tokenNewAddress": "Une nouvelle adresse demande de saisir à nouveau le jeton d'API." } }, "admin": { @@ -10147,6 +10176,70 @@ "body": "La lecture par IA envoie ceci à votre service d'IA hors de ce serveur. Autoriser HealthLog à envoyer des documents, des résultats d'analyses et des descriptions de médicaments pour lecture ?", "grant": "Autoriser la lecture", "error": "Le consentement n'a pas pu être enregistré. Veuillez réessayer." + }, + "sourcePicker": { + "open": "Importer depuis une archive", + "openNamed": "Importer depuis {name}", + "title": "Importer depuis {name}", + "description": "Cherchez par nom, puis cochez les documents que HealthLog doit copier.", + "sourceLabel": "Archive", + "searchLabel": "Nom du document", + "searchPlaceholder": "Chercher par nom", + "tagLabel": "Étiquette", + "tagAny": "Toutes les étiquettes", + "fromLabel": "Du", + "toLabel": "Au", + "clearFilters": "Effacer l'étiquette et les dates", + "rangeInvalid": "La date de fin précède la date de début.", + "empty": "{name} n'a aucun document à afficher.", + "noMatch": "Aucun document ne correspond.", + "loadMore": "Charger plus", + "stateImported": "Dans HealthLog", + "stateDeleted": "Supprimé dans HealthLog", + "selectedCount": "{count} sélectionnés sur {max} au plus", + "importAction": "Importer {count}", + "importing": "Import en cours, un document après l'autre…", + "done": "Terminé", + "summary": "Importés : {imported}. Déjà présents : {present}. Non importés : {failed}.", + "stopped": "Arrêté : {reason}", + "result": { + "waiting": "En attente", + "working": "Import en cours…", + "imported": "Importé", + "importedLinked": "Importé et lié", + "duplicate": "Déjà dans HealthLog", + "duplicateLinked": "Déjà dans HealthLog, désormais lié", + "deleted": "Supprimé dans HealthLog, laissé tel quel" + }, + "errors": { + "unavailable": "Les archives de documents ne sont pas activées sur ce serveur.", + "notConnected": "Cette archive n'est pas connectée. Connectez-la dans Réglages, Intégrations.", + "originNotAllowed": "Cette adresse ne figure pas dans la liste des archives autorisées du serveur. Demandez à l'administration de l'ajouter.", + "unreachable": "L'archive est injoignable. Vérifiez qu'elle fonctionne et que l'adresse est juste.", + "redirected": "L'archive a redirigé la requête. Saisissez l'adresse vers laquelle elle redirige.", + "authRefused": "L'archive a refusé le jeton. Vérifiez le jeton et ses droits.", + "versionTooOld": "L'archive est trop ancienne. Paperless-ngx 2.16 ou plus récent est nécessaire.", + "badResponse": "La réponse de l'archive est illisible. Vérifiez que l'adresse pointe vers l'archive elle-même.", + "notFound": "L'archive ne connaît pas ce document, ou l'identifiant d'organisation est faux ou n'est pas le vôtre.", + "linkTargetNotFound": "L'élément auquel lier le document n'existe plus.", + "rateLimited": "Trop de requêtes. Patientez une minute et réessayez.", + "browserOnly": "Les archives de documents ne s'utilisent que dans le navigateur.", + "invalidAddress": "Saisissez l'adresse complète, commençant par https:// ou http://.", + "organizationRequired": "Saisissez l'identifiant d'organisation Papra.", + "tokenRequired": "Saisissez le jeton d'API. Une nouvelle adresse le demande à nouveau.", + "fileTooLarge": "Le fichier dépasse la taille acceptée par HealthLog.", + "quotaExceeded": "Votre espace de documents est plein.", + "fileType": "HealthLog n'accepte pas ce type de fichier.", + "uploadLimit": "La limite horaire d'envois est atteinte. Réessayez plus tard.", + "uploadBusy": "D'autres envois sont en cours. Réessayez dans un instant.", + "moduleDisabled": "Les documents sont désactivés dans les réglages.", + "generic": "Un problème est survenu. Réessayez.", + "permissionMissing": "La clé Papra ne peut pas lire les étiquettes. Donnez-lui le droit tags:read." + }, + "pickMore": "En choisir d'autres", + "finishedHint": "Cherchez à nouveau ou changez un filtre pour choisir d'autres documents.", + "maxReached": "Vous pouvez importer au plus {max} documents à la fois.", + "noMatchHint": "Essayez une partie du nom, ou effacez l'étiquette et les dates." } }, "selfDescription": { diff --git a/messages/it.json b/messages/it.json index ae8310f2d..802991bbc 100644 --- a/messages/it.json +++ b/messages/it.json @@ -6698,6 +6698,35 @@ "showChecklist": "Mostra di nuovo la lista", "checklistShown": "La lista è di nuovo nella dashboard.", "checklistNothingHidden": "La lista non era nascosta; si apre nella dashboard." + }, + "documentSources": { + "title": "Archivi di documenti", + "description": "Cerca in Paperless-ngx o Papra da HealthLog e importa i documenti che scegli.", + "explainer": "Un pulsante di importazione compare ovunque colleghi documenti: nelle visite, nelle vaccinazioni, nelle condizioni e in Documenti. HealthLog conserva una propria copia cifrata di ogni documento importato e interroga il tuo archivio solo quando cerchi o importi.", + "systemLabel": "Archivio", + "statusConnected": "Collegato", + "statusNotConnected": "Non collegato", + "connectedSince": "Collegato, ultimo controllo il {date}", + "originRemoved": "Questo indirizzo non è più nell'elenco degli archivi consentiti dal server. Chiedi all'amministrazione o salva un altro indirizzo.", + "baseUrl": "Indirizzo", + "allowedOrigins": "Consentito su questo server: {origins}", + "organizationId": "Id organizzazione", + "organizationHelp": "La parte dopo /organizations/ nella barra degli indirizzi di Papra. Gli id organizzazione di Papra hanno la forma org_ seguita da 24 lettere e cifre.", + "token": "Token API", + "tokenSaved": "Salvato per questo indirizzo. Lascia vuoto per mantenerlo.", + "tokenHelpPaperless": "In Paperless-ngx: Il mio profilo, token API.", + "tokenHelpPapra": "In Papra: una chiave API con documents:read e tags:read.", + "save": "Salva e prova", + "saving": "Prova in corso…", + "saved": "Collegato. L'archivio ha risposto.", + "test": "Prova", + "testOk": "L'archivio ha risposto in {latency} ms.", + "disconnect": "Scollega", + "disconnectTitle": "Scollegare {name}?", + "disconnectDescription": "HealthLog dimentica indirizzo e token. I documenti già importati restano in HealthLog.", + "unavailableExplainer": "Questo server non consente più gli archivi di documenti, quindi il collegamento qui sotto non si può usare. Puoi comunque rimuoverlo.", + "moduleOffExplainer": "I documenti sono disattivati, quindi il collegamento qui sotto non viene usato. Puoi comunque rimuoverlo.", + "tokenNewAddress": "Un nuovo indirizzo richiede di nuovo il token API." } }, "admin": { @@ -10147,6 +10176,70 @@ "body": "La lettura con IA invia questo contenuto al tuo servizio di IA fuori da questo server. Consentire a HealthLog di inviare documenti, referti di laboratorio e descrizioni di farmaci per la lettura?", "grant": "Consenti lettura", "error": "Non è stato possibile salvare il consenso. Riprova." + }, + "sourcePicker": { + "open": "Importa da archivio", + "openNamed": "Importa da {name}", + "title": "Importa da {name}", + "description": "Cerca per nome, poi spunta i documenti che HealthLog deve copiare.", + "sourceLabel": "Archivio", + "searchLabel": "Nome del documento", + "searchPlaceholder": "Cerca per nome", + "tagLabel": "Etichetta", + "tagAny": "Qualsiasi etichetta", + "fromLabel": "Dal", + "toLabel": "Al", + "clearFilters": "Azzera etichetta e date", + "rangeInvalid": "La data finale è precedente a quella iniziale.", + "empty": "{name} non ha documenti da mostrare.", + "noMatch": "Nessun documento corrisponde.", + "loadMore": "Carica altri", + "stateImported": "Già in HealthLog", + "stateDeleted": "Eliminato in HealthLog", + "selectedCount": "{count} selezionati su un massimo di {max}", + "importAction": "Importa {count}", + "importing": "Importazione in corso, un documento alla volta…", + "done": "Fatto", + "summary": "Importati: {imported}. Già presenti: {present}. Non importati: {failed}.", + "stopped": "Interrotto: {reason}", + "result": { + "waiting": "In attesa", + "working": "Importazione…", + "imported": "Importato", + "importedLinked": "Importato e collegato", + "duplicate": "Già in HealthLog", + "duplicateLinked": "Già in HealthLog, ora collegato", + "deleted": "Eliminato in HealthLog, lasciato così" + }, + "errors": { + "unavailable": "Gli archivi di documenti non sono abilitati su questo server.", + "notConnected": "Questo archivio non è collegato. Collegalo in Impostazioni, Integrazioni.", + "originNotAllowed": "Questo indirizzo non è nell'elenco degli archivi consentiti dal server. Chiedi all'amministrazione di aggiungerlo.", + "unreachable": "L'archivio non è raggiungibile. Verifica che sia in funzione e che l'indirizzo sia corretto.", + "redirected": "L'archivio ha reindirizzato la richiesta. Inserisci l'indirizzo verso cui reindirizza.", + "authRefused": "L'archivio ha rifiutato il token. Controlla il token e i suoi permessi.", + "versionTooOld": "L'archivio è troppo vecchio. Serve Paperless-ngx 2.16 o successivo.", + "badResponse": "Non è stato possibile leggere la risposta dell'archivio. Verifica che l'indirizzo punti all'archivio stesso.", + "notFound": "L'archivio non ha questo documento, oppure l'id dell'organizzazione è sbagliato o non è tuo.", + "linkTargetNotFound": "L'elemento da collegare non esiste più.", + "rateLimited": "Troppe richieste. Attendi un minuto e riprova.", + "browserOnly": "Gli archivi di documenti si usano solo nel browser.", + "invalidAddress": "Inserisci l'indirizzo completo, che inizia con https:// o http://.", + "organizationRequired": "Inserisci l'id dell'organizzazione Papra.", + "tokenRequired": "Inserisci il token API. Un nuovo indirizzo lo richiede di nuovo.", + "fileTooLarge": "Il file è più grande di quanto HealthLog accetti.", + "quotaExceeded": "Lo spazio per i documenti è pieno.", + "fileType": "HealthLog non accetta questo tipo di file.", + "uploadLimit": "È stato raggiunto il limite orario di caricamenti. Riprova più tardi.", + "uploadBusy": "Altri caricamenti sono ancora in corso. Riprova tra un momento.", + "moduleDisabled": "I documenti sono disattivati nelle impostazioni.", + "generic": "Qualcosa è andato storto. Riprova.", + "permissionMissing": "La chiave Papra non può leggere le etichette. Dalle il permesso tags:read." + }, + "pickMore": "Scegline altri", + "finishedHint": "Cerca di nuovo o cambia un filtro per scegliere altri documenti.", + "maxReached": "Puoi importare al massimo {max} documenti alla volta.", + "noMatchHint": "Prova con parte del nome, oppure azzera etichetta e date." } }, "selfDescription": { diff --git a/messages/ko.json b/messages/ko.json index 674b6c9a3..ace3dd8ac 100644 --- a/messages/ko.json +++ b/messages/ko.json @@ -6698,6 +6698,35 @@ "showChecklist": "체크리스트 다시 표시", "checklistShown": "체크리스트가 대시보드에 다시 표시됩니다.", "checklistNothingHidden": "체크리스트는 숨겨져 있지 않았습니다. 대시보드에서 열립니다." + }, + "documentSources": { + "title": "문서 보관함", + "description": "HealthLog에서 Paperless-ngx나 Papra를 검색하고 선택한 문서를 가져옵니다.", + "explainer": "문서를 연결하는 곳마다 가져오기 버튼이 나타납니다: 진료, 예방접종, 질환, 그리고 문서 페이지. HealthLog는 가져온 문서마다 자체 암호화 사본을 보관하며, 검색하거나 가져올 때만 보관함에 요청합니다.", + "systemLabel": "보관함", + "statusConnected": "연결됨", + "statusNotConnected": "연결 안 됨", + "connectedSince": "연결됨, 마지막 확인 {date}", + "originRemoved": "이 주소는 더 이상 서버의 허용된 보관함 목록에 없습니다. 관리자에게 문의하거나 다른 주소를 저장하세요.", + "baseUrl": "주소", + "allowedOrigins": "이 서버에서 허용됨: {origins}", + "organizationId": "조직 ID", + "organizationHelp": "Papra 주소 표시줄에서 /organizations/ 다음 부분입니다. Papra 조직 ID는 org_ 뒤에 영문자와 숫자 24자가 오는 형식입니다.", + "token": "API 토큰", + "tokenSaved": "이 주소에 저장됨. 유지하려면 비워 두세요.", + "tokenHelpPaperless": "Paperless-ngx: 내 프로필의 API 토큰.", + "tokenHelpPapra": "Papra: documents:read와 tags:read 권한이 있는 API 키.", + "save": "저장하고 테스트", + "saving": "테스트 중…", + "saved": "연결되었습니다. 보관함이 응답했습니다.", + "test": "테스트", + "testOk": "보관함이 {latency}ms 만에 응답했습니다.", + "disconnect": "연결 해제", + "disconnectTitle": "{name} 연결을 해제할까요?", + "disconnectDescription": "HealthLog가 주소와 토큰을 잊습니다. 이미 가져온 문서는 HealthLog에 남습니다.", + "unavailableExplainer": "이 서버는 더 이상 문서 보관함을 허용하지 않아 아래 연결을 사용할 수 없습니다. 연결은 여전히 삭제할 수 있습니다.", + "moduleOffExplainer": "문서 기능이 꺼져 있어 아래 연결은 사용되지 않습니다. 연결은 여전히 삭제할 수 있습니다.", + "tokenNewAddress": "새 주소에는 API 토큰을 다시 입력해야 합니다." } }, "admin": { @@ -10147,6 +10176,70 @@ "body": "AI로 읽으면 이 내용이 이 서버 밖의 AI 서비스로 전송됩니다. HealthLog가 문서, 검사 결과지, 약 설명을 읽기용으로 보내도록 허용하시겠습니까?", "grant": "읽기 허용", "error": "동의를 저장하지 못했습니다. 다시 시도해 주세요." + }, + "sourcePicker": { + "open": "보관함에서 가져오기", + "openNamed": "{name}에서 가져오기", + "title": "{name}에서 가져오기", + "description": "이름으로 검색한 뒤 HealthLog로 복사할 문서를 선택하세요.", + "sourceLabel": "보관함", + "searchLabel": "문서 이름", + "searchPlaceholder": "이름으로 검색", + "tagLabel": "태그", + "tagAny": "모든 태그", + "fromLabel": "시작일", + "toLabel": "종료일", + "clearFilters": "태그와 날짜 지우기", + "rangeInvalid": "종료일이 시작일보다 앞섭니다.", + "empty": "{name}에 표시할 문서가 없습니다.", + "noMatch": "일치하는 문서가 없습니다.", + "loadMore": "더 불러오기", + "stateImported": "HealthLog에 있음", + "stateDeleted": "HealthLog에서 삭제됨", + "selectedCount": "최대 {max}개 중 {count}개 선택", + "importAction": "{count}개 가져오기", + "importing": "문서를 하나씩 가져오는 중…", + "done": "완료", + "summary": "가져옴: {imported}. 이미 있음: {present}. 가져오지 못함: {failed}.", + "stopped": "중단됨: {reason}", + "result": { + "waiting": "대기 중", + "working": "가져오는 중…", + "imported": "가져옴", + "importedLinked": "가져오고 연결함", + "duplicate": "이미 HealthLog에 있음", + "duplicateLinked": "이미 HealthLog에 있음, 이제 연결됨", + "deleted": "HealthLog에서 삭제됨, 그대로 둠" + }, + "errors": { + "unavailable": "이 서버에서는 문서 보관함이 활성화되어 있지 않습니다.", + "notConnected": "이 보관함은 연결되어 있지 않습니다. 설정의 통합에서 연결하세요.", + "originNotAllowed": "이 주소는 서버의 허용된 보관함 목록에 없습니다. 관리자에게 추가를 요청하세요.", + "unreachable": "보관함에 연결할 수 없습니다. 실행 중인지, 주소가 맞는지 확인하세요.", + "redirected": "보관함이 요청을 리디렉션했습니다. 리디렉션되는 주소를 입력하세요.", + "authRefused": "보관함이 토큰을 거부했습니다. 토큰과 권한을 확인하세요.", + "versionTooOld": "보관함 버전이 너무 오래되었습니다. Paperless-ngx 2.16 이상이 필요합니다.", + "badResponse": "보관함의 응답을 읽을 수 없습니다. 주소가 보관함 자체를 가리키는지 확인하세요.", + "notFound": "보관함에 해당 문서가 없거나, 조직 ID가 잘못되었거나 본인의 조직이 아닙니다.", + "linkTargetNotFound": "연결할 기록이 더 이상 존재하지 않습니다.", + "rateLimited": "요청이 너무 많습니다. 1분 후에 다시 시도하세요.", + "browserOnly": "문서 보관함은 브라우저에서만 사용할 수 있습니다.", + "invalidAddress": "https:// 또는 http://로 시작하는 전체 주소를 입력하세요.", + "organizationRequired": "Papra 조직 ID를 입력하세요.", + "tokenRequired": "API 토큰을 입력하세요. 새 주소에는 다시 필요합니다.", + "fileTooLarge": "파일이 HealthLog가 허용하는 크기보다 큽니다.", + "quotaExceeded": "문서 저장 공간이 가득 찼습니다.", + "fileType": "HealthLog는 이 파일 형식을 받지 않습니다.", + "uploadLimit": "시간당 업로드 한도에 도달했습니다. 나중에 다시 시도하세요.", + "uploadBusy": "다른 업로드가 아직 진행 중입니다. 잠시 후 다시 시도하세요.", + "moduleDisabled": "설정에서 문서 기능이 꺼져 있습니다.", + "generic": "문제가 발생했습니다. 다시 시도하세요.", + "permissionMissing": "Papra 키가 태그를 읽을 수 없습니다. tags:read 권한을 부여하세요." + }, + "pickMore": "더 고르기", + "finishedHint": "다시 검색하거나 필터를 바꿔 문서를 더 고르세요.", + "maxReached": "한 번에 최대 {max}개 문서를 가져올 수 있습니다.", + "noMatchHint": "이름의 일부로 검색하거나 태그와 날짜를 지워 보세요." } }, "selfDescription": { diff --git a/messages/pl.json b/messages/pl.json index a2c23d8b3..fef668a57 100644 --- a/messages/pl.json +++ b/messages/pl.json @@ -6698,6 +6698,35 @@ "showChecklist": "Pokaż listę ponownie", "checklistShown": "Lista wróciła na panel.", "checklistNothingHidden": "Lista nie była ukryta; otwiera się na panelu." + }, + "documentSources": { + "title": "Archiwa dokumentów", + "description": "Przeszukuj Paperless-ngx lub Papra z poziomu HealthLog i importuj wybrane dokumenty.", + "explainer": "Przycisk importu pojawia się wszędzie tam, gdzie wiążesz dokumenty: przy wizytach, szczepieniach i schorzeniach oraz w Dokumentach. HealthLog przechowuje własną zaszyfrowaną kopię każdego zaimportowanego dokumentu i pyta archiwum tylko wtedy, gdy szukasz lub importujesz.", + "systemLabel": "Archiwum", + "statusConnected": "Połączono", + "statusNotConnected": "Nie połączono", + "connectedSince": "Połączono, ostatnio sprawdzono {date}", + "originRemoved": "Tego adresu nie ma już na liście dozwolonych archiwów serwera. Zapytaj administrację albo zapisz inny adres.", + "baseUrl": "Adres", + "allowedOrigins": "Dozwolone na tym serwerze: {origins}", + "organizationId": "Identyfikator organizacji", + "organizationHelp": "Część po /organizations/ w pasku adresu Papra. Identyfikatory organizacji Papra mają postać org_ i 24 litery oraz cyfry.", + "token": "Token API", + "tokenSaved": "Zapisano dla tego adresu. Zostaw puste, aby go zachować.", + "tokenHelpPaperless": "W Paperless-ngx: Mój profil, token API.", + "tokenHelpPapra": "W Papra: klucz API z uprawnieniami documents:read i tags:read.", + "save": "Zapisz i sprawdź", + "saving": "Sprawdzanie…", + "saved": "Połączono. Archiwum odpowiedziało.", + "test": "Sprawdź", + "testOk": "Archiwum odpowiedziało w {latency} ms.", + "disconnect": "Rozłącz", + "disconnectTitle": "Rozłączyć {name}?", + "disconnectDescription": "HealthLog zapomina adres i token. Zaimportowane już dokumenty zostają w HealthLog.", + "unavailableExplainer": "Ten serwer nie pozwala już na archiwa dokumentów, więc poniższego połączenia nie da się użyć. Nadal możesz je usunąć.", + "moduleOffExplainer": "Dokumenty są wyłączone, więc poniższe połączenie nie jest używane. Nadal możesz je usunąć.", + "tokenNewAddress": "Nowy adres wymaga ponownego podania tokenu API." } }, "admin": { @@ -10147,6 +10176,70 @@ "body": "Odczyt przez AI wysyła to do Twojej usługi AI poza tym serwerem. Czy HealthLog może wysyłać dokumenty, wyniki badań i opisy leków do odczytu?", "grant": "Zezwól na odczyt", "error": "Nie udało się zapisać zgody. Spróbuj ponownie." + }, + "sourcePicker": { + "open": "Importuj z archiwum", + "openNamed": "Importuj z {name}", + "title": "Import z {name}", + "description": "Szukaj po nazwie, a potem zaznacz dokumenty, które HealthLog ma skopiować.", + "sourceLabel": "Archiwum", + "searchLabel": "Nazwa dokumentu", + "searchPlaceholder": "Szukaj po nazwie", + "tagLabel": "Tag", + "tagAny": "Dowolny tag", + "fromLabel": "Od", + "toLabel": "Do", + "clearFilters": "Wyczyść tag i daty", + "rangeInvalid": "Data końcowa jest wcześniejsza niż początkowa.", + "empty": "{name} nie ma dokumentów do pokazania.", + "noMatch": "Brak pasujących dokumentów.", + "loadMore": "Wczytaj więcej", + "stateImported": "W HealthLog", + "stateDeleted": "Usunięty w HealthLog", + "selectedCount": "Wybrano {count} z maksymalnie {max}", + "importAction": "Importuj ({count})", + "importing": "Importowanie, dokument po dokumencie…", + "done": "Gotowe", + "summary": "Zaimportowane: {imported}. Już były: {present}. Niezaimportowane: {failed}.", + "stopped": "Zatrzymano: {reason}", + "result": { + "waiting": "Czeka", + "working": "Importowanie…", + "imported": "Zaimportowano", + "importedLinked": "Zaimportowano i powiązano", + "duplicate": "Już jest w HealthLog", + "duplicateLinked": "Już jest w HealthLog, teraz powiązany", + "deleted": "Usunięty w HealthLog, pozostaje usunięty" + }, + "errors": { + "unavailable": "Archiwa dokumentów nie są włączone na tym serwerze.", + "notConnected": "To archiwum nie jest połączone. Połącz je w Ustawieniach, Integracje.", + "originNotAllowed": "Tego adresu nie ma na liście dozwolonych archiwów serwera. Poproś administrację o jego dodanie.", + "unreachable": "Nie udało się połączyć z archiwum. Sprawdź, czy działa i czy adres jest poprawny.", + "redirected": "Archiwum przekierowało żądanie. Wpisz adres, na który przekierowuje.", + "authRefused": "Archiwum odrzuciło token. Sprawdź token i jego uprawnienia.", + "versionTooOld": "Archiwum jest za stare. Potrzebny jest Paperless-ngx 2.16 lub nowszy.", + "badResponse": "Nie udało się odczytać odpowiedzi archiwum. Sprawdź, czy adres wskazuje na samo archiwum.", + "notFound": "Archiwum nie ma takiego dokumentu albo identyfikator organizacji jest błędny lub nie należy do ciebie.", + "linkTargetNotFound": "Wpis, z którym miało nastąpić powiązanie, już nie istnieje.", + "rateLimited": "Za dużo żądań. Odczekaj minutę i spróbuj ponownie.", + "browserOnly": "Z archiwów dokumentów można korzystać tylko w przeglądarce.", + "invalidAddress": "Wpisz pełny adres, zaczynający się od https:// lub http://.", + "organizationRequired": "Wpisz identyfikator organizacji Papra.", + "tokenRequired": "Wpisz token API. Nowy adres wymaga go ponownie.", + "fileTooLarge": "Plik jest większy, niż przyjmuje HealthLog.", + "quotaExceeded": "Miejsce na dokumenty jest pełne.", + "fileType": "HealthLog nie przyjmuje tego typu pliku.", + "uploadLimit": "Osiągnięto godzinny limit przesyłania. Spróbuj później.", + "uploadBusy": "Trwają jeszcze inne przesyłania. Spróbuj za chwilę.", + "moduleDisabled": "Dokumenty są wyłączone w ustawieniach.", + "generic": "Coś poszło nie tak. Spróbuj ponownie.", + "permissionMissing": "Klucz Papra nie może odczytać tagów. Nadaj mu uprawnienie tags:read." + }, + "pickMore": "Wybierz więcej", + "finishedHint": "Wyszukaj ponownie lub zmień filtr, aby wybrać więcej dokumentów.", + "maxReached": "Możesz zaimportować naraz najwyżej {max} dokumentów.", + "noMatchHint": "Spróbuj części nazwy albo wyczyść tag i daty." } }, "selfDescription": { diff --git a/prisma/migrations/0356_document_source_connections/migration.sql b/prisma/migrations/0356_document_source_connections/migration.sql new file mode 100644 index 000000000..f50be788c --- /dev/null +++ b/prisma/migrations/0356_document_source_connections/migration.sql @@ -0,0 +1,46 @@ +-- Document picker (#1038). One connection per person per system (Paperless-ngx +-- or Papra) with the API token encrypted at rest. The picker searches the +-- source and imports picked documents on request only; nothing polls. +CREATE TABLE "document_source_connections" ( + "id" TEXT NOT NULL, + "user_id" TEXT NOT NULL, + "system" VARCHAR(16) NOT NULL, + "base_url" VARCHAR(2048) NOT NULL, + "organization_id" VARCHAR(128), + "token_encrypted" TEXT NOT NULL, + "last_verified_at" TIMESTAMP(3), + "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" TIMESTAMP(3) NOT NULL, + CONSTRAINT "document_source_connections_pkey" PRIMARY KEY ("id") +); + +CREATE UNIQUE INDEX "document_source_connections_user_id_system_key" + ON "document_source_connections" ("user_id", "system"); + +ALTER TABLE "document_source_connections" + ADD CONSTRAINT "document_source_connections_user_id_fkey" + FOREIGN KEY ("user_id") REFERENCES "users"("id") ON DELETE CASCADE ON UPDATE CASCADE; + +-- Source instance (#1038 review): the same id in two Paperless-ngx (or two +-- Papra) instances is two documents. A nullable origin joins every source key; +-- keys stored without one (v1.39.2 imports) keep matching any instance, which +-- the application decides at lookup time. NULLS NOT DISTINCT keeps two keys +-- without an instance colliding exactly as before (PG15+; the stack runs 16). +ALTER TABLE "inbound_documents" ADD COLUMN "source_instance" VARCHAR(512); +DROP INDEX IF EXISTS "inbound_documents_user_source_key"; +CREATE UNIQUE INDEX "inbound_documents_user_source_key" + ON "inbound_documents" ("user_id", "source_system", "source_instance", "source_id") + NULLS NOT DISTINCT + WHERE "source_id" IS NOT NULL; + +ALTER TABLE "document_source_aliases" ADD COLUMN "source_instance" VARCHAR(512); +DROP INDEX IF EXISTS "document_source_aliases_user_id_source_system_source_id_key"; +CREATE UNIQUE INDEX "document_source_aliases_user_id_source_system_source_instance_source_id_key" + ON "document_source_aliases" ("user_id", "source_system", "source_instance", "source_id") + NULLS NOT DISTINCT; + +ALTER TABLE "document_import_keys" ADD COLUMN "source_instance" VARCHAR(512); +DROP INDEX IF EXISTS "document_import_keys_user_id_source_system_source_id_key"; +CREATE UNIQUE INDEX "document_import_keys_user_id_source_system_source_instance_source_id_key" + ON "document_import_keys" ("user_id", "source_system", "source_instance", "source_id") + NULLS NOT DISTINCT; diff --git a/prisma/schema.prisma b/prisma/schema.prisma index 7d725e8d3..6c6c06bca 100644 --- a/prisma/schema.prisma +++ b/prisma/schema.prisma @@ -928,6 +928,9 @@ model User { // v1.39.2 — further source keys an import answered with an existing // document (same bytes under another key). See DocumentSourceAlias. documentSourceAliases DocumentSourceAlias[] + // v1.39.3 — the person's own connection to a Paperless-ngx or Papra + // instance for the document picker. See DocumentSourceConnection. + documentSourceConnections DocumentSourceConnection[] extractedFacts ExtractedFact[] documentConditionLinks DocumentConditionLink[] documentContentIndexes DocumentContentIndex[] @@ -8008,15 +8011,24 @@ model InboundDocument { /// and the Paperless workflow recipe; shown on the detail sheet. sourceSystem String? @map("source_system") @db.VarChar(16) /// v1.39.2 (#1038) — the document's id in `sourceSystem`. Together with - /// `userId` + `sourceSystem` it is unique across live AND tombstoned rows - /// (raw SQL in migration 0353, not expressible in Prisma): - /// CREATE UNIQUE INDEX inbound_documents_user_source_key - /// ON inbound_documents (user_id, source_system, source_id) - /// WHERE source_id IS NOT NULL; + /// `userId`, `sourceSystem` and `sourceInstance` it is unique across live + /// AND tombstoned rows (raw SQL in migrations 0353 and 0356, not expressible + /// in Prisma; see `sourceInstance` for the current definition). /// Including tombstones is the point: re-running an importer must not bring /// back a document the person deleted. The purge carries the key over to /// `DocumentImportKey` when it removes the row. sourceId String? @map("source_id") @db.VarChar(128) + /// v1.39.3 (#1038) — which instance of `sourceSystem` the document came + /// from: the normalised origin (`scheme://host[:port]`) of the Paperless-ngx + /// or Papra it was fetched from, so id 123 on one Paperless and id 123 on + /// another are two keys. Plaintext; an address, not health data. Null for a + /// key sent without one (every v1.39.2 import, an older script, a workflow + /// that does not send it): such a key matches any instance of its system, + /// and a lookup without an instance matches any stored instance. Part of the + /// source-key unique index since migration 0356, which rebuilt it as + /// (user_id, source_system, source_instance, source_id) NULLS NOT DISTINCT + /// WHERE source_id IS NOT NULL. + sourceInstance String? @map("source_instance") @db.VarChar(512) /// v1.39.2 (#1038) — the upload asked for `aiRead=defer`: nothing may send /// this document to an AI provider on its own (the summary catch-up, the /// summary job, lab auto-staging all skip it). Cleared when the person @@ -8078,18 +8090,24 @@ model InboundDocument { /// own, so deleting the document keeps every key that ever reached it deleted. /// Cascades with the document; the purge copies it first. model DocumentSourceAlias { - id String @id @default(cuid()) - userId String @map("user_id") - documentId String @map("document_id") + id String @id @default(cuid()) + userId String @map("user_id") + documentId String @map("document_id") /// Same closed vocabulary as `InboundDocument.sourceSystem`. - sourceSystem String @map("source_system") @db.VarChar(16) - sourceId String @map("source_id") @db.VarChar(128) - createdAt DateTime @default(now()) @map("created_at") + sourceSystem String @map("source_system") @db.VarChar(16) + sourceId String @map("source_id") @db.VarChar(128) + /// v1.39.3 — the source instance, as on `InboundDocument.sourceInstance` + /// (null matches any instance of the system). + sourceInstance String? @map("source_instance") @db.VarChar(512) + createdAt DateTime @default(now()) @map("created_at") user User @relation(fields: [userId], references: [id], onDelete: Cascade) document InboundDocument @relation(fields: [documentId], references: [id], onDelete: Cascade) - @@unique([userId, sourceSystem, sourceId]) + // NULLS NOT DISTINCT in migration 0356 (hand-written; Prisma would + // regenerate it NULLS DISTINCT): a key stored without an instance still + // collides with the same key without one. + @@unique([userId, sourceSystem, sourceInstance, sourceId]) @@index([documentId]) @@map("document_source_aliases") } @@ -8101,19 +8119,62 @@ model DocumentSourceAlias { /// "deleted" instead of storing the document again. Written by the purge only; /// read by the upload route's source-key check. Holds no document content. model DocumentImportKey { - id String @id @default(cuid()) - userId String @map("user_id") + id String @id @default(cuid()) + userId String @map("user_id") /// Same closed vocabulary as `InboundDocument.sourceSystem`. - sourceSystem String @map("source_system") @db.VarChar(16) - sourceId String @map("source_id") @db.VarChar(128) - createdAt DateTime @default(now()) @map("created_at") + sourceSystem String @map("source_system") @db.VarChar(16) + sourceId String @map("source_id") @db.VarChar(128) + /// v1.39.3 — the source instance, as on `InboundDocument.sourceInstance` + /// (null matches any instance of the system). + sourceInstance String? @map("source_instance") @db.VarChar(512) + createdAt DateTime @default(now()) @map("created_at") user User @relation(fields: [userId], references: [id], onDelete: Cascade) - @@unique([userId, sourceSystem, sourceId]) + // NULLS NOT DISTINCT in migration 0356 (hand-written; Prisma would + // regenerate it NULLS DISTINCT): a key stored without an instance still + // collides with the same key without one. + @@unique([userId, sourceSystem, sourceInstance, sourceId]) @@map("document_import_keys") } +/// v1.39.3 (#1038) — one person's connection to a Paperless-ngx or Papra +/// instance, used by the document picker to search that system by name, tag +/// and date and import the documents the person picks. One per system per +/// person. Only reachable when the operator lists the instance's origin in +/// `DOCUMENT_SOURCE_ORIGINS`; the origin is re-checked on every request, so +/// removing it from the list cuts the connection off without touching the +/// row. No background job reads it: every request to the source is one the +/// person started. Not carried in backups (a third-party credential for a +/// service on the operator's network; see `NOT_IN_BACKUP_MODELS`); removed by +/// Delete all data and with the account. +model DocumentSourceConnection { + id String @id @default(cuid()) + userId String @map("user_id") + /// `PAPERLESS` | `PAPRA` (`DOCUMENT_PICKER_SYSTEMS` in + /// `src/lib/documents/sources/types.ts`). + system String @map("system") @db.VarChar(16) + /// Base address of the instance, origin plus an optional path. Plaintext: + /// its origin is compared with the operator's list on every use, and the + /// settings card shows it. Not health data. + baseUrl String @map("base_url") @db.VarChar(2048) + /// Papra organization id; null for Paperless-ngx. + organizationId String? @map("organization_id") @db.VarChar(128) + /// The API token (Paperless-ngx) or API key (Papra), AES-256-GCM via + /// `encrypt()`. No route returns it, in whole or in part. + tokenEncrypted String @map("token_encrypted") + /// Last time the source answered a test with this token: the save's live + /// check or the settings card's Test button. Shown on the card. + lastVerifiedAt DateTime? @map("last_verified_at") + createdAt DateTime @default(now()) @map("created_at") + updatedAt DateTime @updatedAt @map("updated_at") + + user User @relation(fields: [userId], references: [id], onDelete: Cascade) + + @@unique([userId, system]) + @@map("document_source_connections") +} + model ExtractedFact { id String @id @default(cuid()) documentId String @map("document_id") diff --git a/scripts/__tests__/import-documents.test.ts b/scripts/__tests__/import-documents.test.ts index 6c5865fa4..754a41e38 100644 --- a/scripts/__tests__/import-documents.test.ts +++ b/scripts/__tests__/import-documents.test.ts @@ -343,7 +343,7 @@ async function papra() { return { status: 200, json: { - documentsCount: 3, + documentsCount: 4, documents: [ { id: "doc_a", @@ -365,6 +365,15 @@ async function papra() { { id: "tag_l", name: "Lab" }, ], }, + { + id: "doc_huge", + name: "Scan.pdf", + originalName: "scan.pdf", + originalSize: 40_000_000, + createdAt: "2025-03-10T12:00:00.000Z", + documentDate: "2020-03-10", + tags: [{ id: "tag_h", name: "Health" }], + }, { id: "doc_old", name: "Old.pdf", @@ -424,6 +433,7 @@ describe("import-documents.mjs — Paperless-ngx", () => { kind: "LAB_RESULT", sourceSystem: "PAPERLESS", sourceId: "1", + sourceInstance: new URL(pl.url).origin, aiRead: "defer", }, }); @@ -664,6 +674,7 @@ describe("import-documents.mjs — Papra", () => { documentDate: "2019-06-01", sourceSystem: "PAPRA", sourceId: "doc_a", + sourceInstance: new URL(pp.url).origin, }); // --ai-read: no deferral field at all. expect(sent[0].aiRead).toBeUndefined(); @@ -675,7 +686,31 @@ describe("import-documents.mjs — Papra", () => { ); expect(pp.seen[0].auth).toBe("Bearer pp_test"); expect(res.stdout).toMatch(/Imported: 1/); - expect(res.stdout).toMatch(/Skipped: 1/); + // doc_b answered 413 with the cap; doc_huge is known to be larger than + // that and is skipped without being downloaded or sent. + expect(res.stdout).toMatch(/Skipped: 2/); expect(res.stdout).toMatch(/larger than HealthLog accepts/); + expect(pp.seen.some((s) => s.path.includes("doc_huge"))).toBe(false); + }); + + it("shows small sizes in KB on a dry run", async () => { + const pp = await papra(); + const res = await runScript( + [ + "papra", + "--papra-url", + pp.url, + "--papra-org", + "org_1", + "--tag", + "Health", + "--since", + "2016-01-01", + "--dry-run", + ], + { PAPRA_TOKEN: "pp_test" }, + ); + expect(res.code).toBe(0); + expect(res.stdout).toMatch(/2\.0 KB\s+Papra doc_a/); }); }); diff --git a/scripts/env-manifest.json b/scripts/env-manifest.json index 11fcfadb5..8992f4abc 100644 --- a/scripts/env-manifest.json +++ b/scripts/env-manifest.json @@ -245,6 +245,17 @@ } ] }, + { + "name": "Document picker", + "description": "Operator grant for the document picker, which lets a person search their own Paperless-ngx or Papra instance and import picked documents. Unset, the picker is off. Optional.", + "required": false, + "variables": [ + { + "name": "DOCUMENT_SOURCE_ORIGINS", + "purpose": "Comma-separated exact origins (scheme://host[:port], no path, no wildcard, no CIDR), e.g. \"https://paperless.example.com,http://papra.lan:1221\". Only listed origins are contacted, public or private; redirects are not followed; link-local, metadata and unspecified addresses stay refused even when listed; a malformed entry is logged once and grants nothing." + } + ] + }, { "name": "Off-host backups", "description": "Required ALL-OR-NONE: any of these without the rest disables backups silently.", diff --git a/scripts/import-documents.mjs b/scripts/import-documents.mjs index 1394136b3..194af9d24 100644 --- a/scripts/import-documents.mjs +++ b/scripts/import-documents.mjs @@ -301,6 +301,7 @@ function paperless(options) { return { system: "PAPERLESS", + instance: new URL(base).origin, label, async *documents() { const query = new URLSearchParams({ @@ -324,7 +325,8 @@ function paperless(options) { tagId = found[0].id; query.set("tags__id__all", String(tagId)); } - if (options.since) query.set("created__date__gte", options.since); + // API version 9 made `created` a plain date: filter on it directly. + if (options.since) query.set("created__gte", options.since); let typeNames = new Map(); if (options.kindMap.size > 0) { @@ -399,6 +401,7 @@ function papra(options) { return { system: "PAPRA", + instance: new URL(options.baseUrl).origin, label, async *documents() { let tagId = null; @@ -526,6 +529,7 @@ function keyQuery(source, doc) { return new URLSearchParams({ sourceSystem: source.system, sourceId: doc.sourceId.slice(0, SOURCE_ID_MAX), + sourceInstance: source.instance, }); } @@ -617,6 +621,9 @@ async function upload(options, source, doc, file) { if (!options.aiRead) form.append("aiRead", "defer"); form.append("sourceSystem", source.system); form.append("sourceId", doc.sourceId.slice(0, SOURCE_ID_MAX)); + // Which Paperless-ngx or Papra this id belongs to: the same id in two + // instances is two documents. HealthLog 1.39.2 ignores the field. + form.append("sourceInstance", source.instance); return form; }, ); @@ -640,6 +647,8 @@ async function upload(options, source, doc, file) { return { outcome: "skipped", reason: `larger than HealthLog accepts${meta.maxFileBytes ? ` (${mib(meta.maxFileBytes)})` : ""}`, + maxFileBytes: + typeof meta.maxFileBytes === "number" ? meta.maxFileBytes : undefined, }; case 415: return { @@ -661,9 +670,10 @@ async function upload(options, source, doc, file) { function mib(bytes) { if (typeof bytes !== "number") return "?"; - return bytes >= 1024 * 1024 * 1024 - ? `${(bytes / 1024 ** 3).toFixed(1)} GB` - : `${(bytes / 1024 ** 2).toFixed(1)} MB`; + if (bytes >= 1024 ** 3) return `${(bytes / 1024 ** 3).toFixed(1)} GB`; + // Under 0.1 MB a size in MB reads as 0.0; say it in KB. + if (bytes < 0.1 * 1024 ** 2) return `${(bytes / 1024).toFixed(1)} KB`; + return `${(bytes / 1024 ** 2).toFixed(1)} MB`; } // ── Run ──────────────────────────────────────────────────────────────────── @@ -698,6 +708,8 @@ async function run(argv, env = process.env) { }; let listed = 0; let totalBytes = 0; + /** HealthLog's per-file limit, once a 413 has said it. */ + let maxFileBytes = null; let unknownSizes = 0; out( @@ -736,6 +748,20 @@ async function run(argv, env = process.env) { continue; } + // HealthLog says its per-file limit only when it refuses a file. Once + // it has, a document the source already says is larger is skipped + // without downloading it. + if ( + maxFileBytes !== null && + typeof doc.size === "number" && + doc.size > maxFileBytes + ) { + const reason = `larger than HealthLog accepts (${mib(maxFileBytes)})`; + tally.skipped.push({ where, reason }); + out(` skipped ${where}: ${reason}`); + continue; + } + const file = await source.download(doc); if (file.skip) { tally.skipped.push({ where, reason: file.skip }); @@ -743,6 +769,9 @@ async function run(argv, env = process.env) { continue; } const result = await upload(options, source, doc, file); + if (typeof result.maxFileBytes === "number") { + maxFileBytes = result.maxFileBytes; + } if (result.outcome === "imported") { tally.imported++; out(` imported ${where}`); diff --git a/scripts/rotate-encryption-key.ts b/scripts/rotate-encryption-key.ts index 0bac08bc9..20fa5124b 100644 --- a/scripts/rotate-encryption-key.ts +++ b/scripts/rotate-encryption-key.ts @@ -328,6 +328,16 @@ async function main() { ); } + // ───── Document picker (String) ───── + // Column: "tokenEncrypted" — a person's Paperless-ngx / Papra API token. + results.push( + await rotateStringColumn( + "DocumentSourceConnection", + "tokenEncrypted", + prisma.documentSourceConnection, + ), + ); + // ───── AppSettings — operator credentials (String) ───── // Columns: "adminAiKeyEncrypted" "webPushVapidPrivateKeyEncrypted" // "adminCodexAccessTokenEncrypted" "adminCodexRefreshTokenEncrypted" diff --git a/src/__tests__/destructive-control-guard.test.ts b/src/__tests__/destructive-control-guard.test.ts index 27a745581..fcf209cfe 100644 --- a/src/__tests__/destructive-control-guard.test.ts +++ b/src/__tests__/destructive-control-guard.test.ts @@ -579,6 +579,13 @@ const REGISTRY: DestructiveEntry[] = [ recovery: "permanent", confirm: ["AlertDialog"], })), + { + file: "components/settings/integrations/document-sources-card.tsx", + destroys: + "the stored Paperless-ngx or Papra address and API token; documents already imported stay in the vault — reconnecting means typing the token again", + recovery: "permanent", + confirm: ["AlertDialog"], + }, ]; // ── Detectors ───────────────────────────────────────────────────────────── diff --git a/src/__tests__/document-sources-owner-only-guard.test.ts b/src/__tests__/document-sources-owner-only-guard.test.ts new file mode 100644 index 000000000..a61741026 --- /dev/null +++ b/src/__tests__/document-sources-owner-only-guard.test.ts @@ -0,0 +1,106 @@ +/** + * The document picker's routes (#1038) are owner-only, cookie-only, and stay + * that way. + * + * A connection is the owner's credential to a system outside HealthLog. A + * delegate or guardian driving it would read the owner's whole Paperless-ngx or + * Papra archive, which no grant covers; a Bearer (the native client, a script) + * has no business managing it either. The rule is in two halves, and this file + * freezes both at every route module under `src/app/api/documents/sources`: + * + * 1. the module calls a bare `requireAuth()` itself (no scope, so a narrow + * token is refused, and a request acting on another record is refused + * 403 `sharing.not_permitted`), and names none of the resolvers that + * would admit a delegate, a guardian, or an actor surface; + * 2. every handler passes the result through `admitDocumentSourceCaller`, + * which refuses any transport but the cookie. + * + * The scan asserts it found the modules it expects, so an empty directory or a + * moved tree fails rather than passing on nothing. + * + * Mutation checks: swap one route's `requireAuth()` for + * `requireRecordAuth("write", "documents")`, or drop its + * `admitDocumentSourceCaller` call, and this file fails naming that module. + */ +import { readFileSync, readdirSync } from "node:fs"; +import { join, relative } from "node:path"; + +import { describe, expect, it } from "vitest"; + +import { stripComments } from "./helpers/source-files"; + +const ROOT = join(process.cwd(), "src/app/api/documents/sources"); + +function routeModules(): string[] { + const out: string[] = []; + const walk = (dir: string) => { + for (const entry of readdirSync(dir, { withFileTypes: true })) { + const full = join(dir, entry.name); + if (entry.isDirectory()) walk(full); + else if (entry.name === "route.ts") out.push(full); + } + }; + walk(ROOT); + return out.map((file) => relative(ROOT, file)).sort(); +} + +const EXPECTED = [ + "[system]/import/route.ts", + "[system]/route.ts", + "[system]/search/route.ts", + "[system]/tags/route.ts", + "[system]/test/route.ts", + "route.ts", +]; + +describe("document-source routes are owner-only and cookie-only", () => { + const modules = routeModules(); + + it("finds exactly the route modules it freezes", () => { + expect(modules).toEqual(EXPECTED); + }); + + it.each(EXPECTED)( + "%s resolves a bare requireAuth and no wider resolver", + (rel) => { + const source = stripComments(readFileSync(join(ROOT, rel), "utf8")); + expect(source).toMatch(/requireAuth\(\s*\)/); + expect(source).not.toMatch(/requireAuth\(\s*[^)\s]/); + for (const wider of [ + "requireRecordAuth", + "requireGuardianAuth", + "requireActorAuth", + "requireCookieAuth", + "requireBearerAuth", + ]) { + expect(source, `${rel} names ${wider}`).not.toContain(wider); + } + }, + ); + + it.each(EXPECTED)( + "%s admits every handler through the cookie check", + (rel) => { + const source = stripComments(readFileSync(join(ROOT, rel), "utf8")); + const handlers = source.match( + /export const (GET|POST|PUT|PATCH|DELETE)\b/g, + ); + expect(handlers?.length ?? 0).toBeGreaterThan(0); + const auths = source.match(/await requireAuth\(\s*\)/g) ?? []; + const admits = source.match(/await admitDocumentSourceCaller\(/g) ?? []; + expect(auths.length).toBe(handlers!.length); + expect(admits.length).toBe(handlers!.length); + }, + ); + + it("the cookie check refuses every transport but the cookie", () => { + const support = stripComments( + readFileSync( + join(process.cwd(), "src/lib/documents/sources/route-support.ts"), + "utf8", + ), + ); + expect(support).toContain('auth.authMethod !== "cookie"'); + expect(support).toContain('"documents.sources.browserOnly"'); + }); +}); diff --git a/src/__tests__/success-affordance-guard.test.ts b/src/__tests__/success-affordance-guard.test.ts index 8cd324ece..d8f561058 100644 --- a/src/__tests__/success-affordance-guard.test.ts +++ b/src/__tests__/success-affordance-guard.test.ts @@ -161,6 +161,13 @@ const PINNED_AFFORDANCES: Record< "src/components/documents/document-card.tsx": { "text-success": 1 }, "src/components/documents/document-detail-sheet.tsx": { "toast.success": 2 }, "src/components/documents/documents-view.tsx": { "toast.success": 3 }, + // The document picker's per-row result: tinted only for the `imported` / + // `duplicate` outcome the import route returned for that one document, i.e. + // a document the vault now holds. The run's headline goes through + // WrittenOutcomeLine; a row sits inside a button, where its

cannot. + "src/components/documents/sources/document-source-picker.tsx": { + "text-success": 1, + }, "src/components/error-details.tsx": { CheckCircle2: 2, "text-success": 1 }, "src/components/illness/episode-menu.tsx": { CheckCircle2: 2, diff --git a/src/app/api/__tests__/ai-route-inventory.ts b/src/app/api/__tests__/ai-route-inventory.ts index 2e49101b2..7e90462ed 100644 --- a/src/app/api/__tests__/ai-route-inventory.ts +++ b/src/app/api/__tests__/ai-route-inventory.ts @@ -67,6 +67,11 @@ export const AI_ROUTES: Readonly> = { capabilities: ["documentAi"], why: "The upload is data and always accepted; only the AI work queued after it follows the capability.", }, + "src/app/api/documents/sources/[system]/import/route.ts": { + kind: "mixed", + capabilities: ["documentAi"], + why: "A picked document is stored whatever AI says; only the summary queued after a fresh insert follows the capability.", + }, "src/app/api/auth/me/documents-auto-ai-read/route.ts": { kind: "mixed", capabilities: ["documentAi"], diff --git a/src/app/api/documents/inbound/__tests__/route.test.ts b/src/app/api/documents/inbound/__tests__/route.test.ts index bcb479701..a83d87578 100644 --- a/src/app/api/documents/inbound/__tests__/route.test.ts +++ b/src/app/api/documents/inbound/__tests__/route.test.ts @@ -51,10 +51,10 @@ vi.mock("@/lib/db", () => { findFirst: vi.fn(), }, documentImportKey: { - findUnique: vi.fn(), + findFirst: vi.fn(), }, documentSourceAlias: { - findUnique: vi.fn(), + findFirst: vi.fn(), createMany: vi.fn(), count: vi.fn(), }, @@ -239,10 +239,10 @@ beforeEach(() => { [] as never, ); vi.mocked(prisma.inboundDocument.findFirst).mockResolvedValue(null as never); - vi.mocked(prisma.documentImportKey.findUnique).mockResolvedValue( + vi.mocked(prisma.documentImportKey.findFirst).mockResolvedValue( null as never, ); - vi.mocked(prisma.documentSourceAlias.findUnique).mockResolvedValue( + vi.mocked(prisma.documentSourceAlias.findFirst).mockResolvedValue( null as never, ); vi.mocked(prisma.documentSourceAlias.createMany).mockResolvedValue({ @@ -783,7 +783,7 @@ describe("POST /api/documents/inbound — document import", () => { }); it("remembers a purged import through the ledger", async () => { - vi.mocked(prisma.documentImportKey.findUnique).mockResolvedValue({ + vi.mocked(prisma.documentImportKey.findFirst).mockResolvedValue({ id: "key-1", } as never); const res = await post( @@ -909,6 +909,8 @@ describe("POST /api/documents/inbound — document import", () => { userId: "user-1", documentId: "doc-manual", sourceSystem: "PAPRA", + // Sent without an instance: the alias matches any Papra. + sourceInstance: null, sourceId: "doc_9", }, ], @@ -936,7 +938,7 @@ describe("POST /api/documents/inbound — document import", () => { }); it("resolves a key held as an alias of a deleted document to deleted", async () => { - vi.mocked(prisma.documentSourceAlias.findUnique).mockResolvedValue({ + vi.mocked(prisma.documentSourceAlias.findFirst).mockResolvedValue({ document: docRow({ id: "doc-gone", deletedAt: new Date() }), } as never); const res = await post( diff --git a/src/app/api/documents/inbound/route.ts b/src/app/api/documents/inbound/route.ts index 70097db7b..f2c6723d4 100644 --- a/src/app/api/documents/inbound/route.ts +++ b/src/app/api/documents/inbound/route.ts @@ -32,7 +32,6 @@ * instruction inside it. Storing it does nothing with its contents. */ import { Buffer } from "node:buffer"; -import { createHash } from "node:crypto"; import { NextResponse } from "next/server"; @@ -51,42 +50,38 @@ import { sanitiseZodIssues, } from "@/lib/api-response"; import { getAiCapability } from "@/lib/ai/capabilities/gate"; -import { auditLog } from "@/lib/auth/audit"; import { prisma } from "@/lib/db"; import { hashQueryTokens } from "@/lib/documents/content-index"; +import { + ingestDocument, + isoDateToUtc, + personalUploadBucket, + UPLOAD_LIMIT_PER_HOUR, + UPLOAD_WINDOW_MS, +} from "@/lib/documents/ingest"; import { loadConditionLinks, loadDocumentEncounterLinks, - narrowOwnedEncounterIds, - narrowOwnedEpisodeIds, } from "@/lib/documents/links"; import { - encryptDocumentContent, serialiseDocument, type SerialisableDocument, } from "@/lib/documents/store"; -import { enqueueDocumentIndex } from "@/lib/jobs/document-index"; -import { enqueueDocumentSummary } from "@/lib/jobs/document-summary"; -import { enqueueDocumentThumbnail } from "@/lib/jobs/document-thumbnail"; import { DOCUMENTS_WRITE_SCOPE } from "@/lib/documents/scopes"; import { checkSourceLookupRateLimit, findSourceKey, - rememberSourceAlias, type SourceKeyMatch, } from "@/lib/documents/source-key"; import { acquireDocumentUploadSlot, - detectDocumentType, resolveDocumentLimits, resolveDocumentUploadLimitPerHour, } from "@/lib/documents/upload-policy"; import { withIdempotency } from "@/lib/idempotency"; import { BodyTooLargeError, readBoundedBody } from "@/lib/labs/ocr-upload"; -import { linkTargets } from "@/lib/links"; import { annotate } from "@/lib/logging/context"; import { requireModuleEnabled } from "@/lib/modules/gate"; -import { isP2002 } from "@/lib/prisma-errors"; import { checkRateLimit, rateLimitHeaders } from "@/lib/rate-limit"; import { documentCreateSchema, @@ -99,13 +94,6 @@ import type { Prisma } from "@/generated/prisma/client"; export const dynamic = "force-dynamic"; -/** - * A store-only upload touches no provider — the only abuse vector is disk, so - * a generous per-user ceiling is enough (extraction has its own tight gate). - */ -const UPLOAD_LIMIT_PER_HOUR = 60; -const UPLOAD_WINDOW_MS = 60 * 60 * 1000; - /** * Multipart envelope allowance on top of the per-file cap: boundaries plus * the small metadata fields (title / kind / documentDate / episodeIds). The @@ -114,11 +102,6 @@ const UPLOAD_WINDOW_MS = 60 * 60 * 1000; const MULTIPART_OVERHEAD_BYTES = 64 * 1024; const UPLOAD_BODY_TIMEOUT_MS = 30_000; -/** Parse a YYYY-MM-DD form field into a UTC midnight Date. */ -function isoDateToUtc(value: string): Date { - return new Date(`${value}T00:00:00.000Z`); -} - /** §3.2 — 413 fileTooLarge with the configured limit in `meta`. */ function fileTooLarge(maxFileBytes: number): NextResponse { return apiError("File is too large.", 413, { @@ -128,14 +111,6 @@ function fileTooLarge(maxFileBytes: number): NextResponse { }); } -/** Internal signal: the quota gate inside the insert transaction tripped. */ -class QuotaExceededError extends Error { - constructor(public readonly usedBytes: number) { - super("Document quota exceeded"); - this.name = "QuotaExceededError"; - } -} - /** What a single upload asked for, carried into the annotations. */ interface UploadContext { userId: string; @@ -299,14 +274,17 @@ async function processUpload( const url = new URL(request.url); const querySystem = url.searchParams.get("sourceSystem"); const queryId = url.searchParams.get("sourceId"); + const queryInstance = url.searchParams.get("sourceInstance"); let queryKey: { sourceSystem: DocumentSourceSystemValue; sourceId: string; + sourceInstance?: string; } | null = null; if (querySystem !== null || queryId !== null) { const parsedKey = documentSourceKeySchema.safeParse({ sourceSystem: querySystem ?? undefined, sourceId: queryId ?? undefined, + sourceInstance: queryInstance ?? undefined, }); if (!parsedKey.success) { return apiValidationError( @@ -337,6 +315,7 @@ async function processUpload( user.id, queryKey.sourceSystem, queryKey.sourceId, + queryKey.sourceInstance ?? null, ); if (match) { return answerSourceKey( @@ -357,7 +336,7 @@ async function processUpload( // draw on. The cookie / wildcard bucket is unchanged. const bucketKey = scoped ? `documents-upload:token:${auth.session.id}` - : `documents-upload:${user.id}`; + : personalUploadBucket(user.id); const rl = await checkRateLimit( bucketKey, scoped ? resolveDocumentUploadLimitPerHour() : UPLOAD_LIMIT_PER_HOUR, @@ -433,6 +412,7 @@ async function processUpload( encounterIds: rawEncounterIds.length > 0 ? rawEncounterIds : undefined, sourceSystem: formData.get("sourceSystem") ?? undefined, sourceId: formData.get("sourceId") ?? undefined, + sourceInstance: formData.get("sourceInstance") ?? undefined, aiRead: formData.get("aiRead") ?? undefined, }); if (!parsed.success) { @@ -450,10 +430,14 @@ async function processUpload( // both they have to agree. const formSystem = parsed.data.sourceSystem ?? null; const formId = formSystem ? (parsed.data.sourceId ?? null) : null; + const formInstance = formId ? (parsed.data.sourceInstance ?? null) : null; if ( queryKey && formSystem !== null && - (formSystem !== queryKey.sourceSystem || formId !== queryKey.sourceId) + (formSystem !== queryKey.sourceSystem || + formId !== queryKey.sourceId || + (formInstance !== null && + formInstance !== (queryKey.sourceInstance ?? null))) ) { return apiError( "The source key in the address and in the form differ.", @@ -463,6 +447,9 @@ async function processUpload( } const sourceSystem = queryKey?.sourceSystem ?? formSystem; const sourceId = queryKey?.sourceId ?? formId; + const sourceInstance = sourceId + ? (queryKey?.sourceInstance ?? formInstance ?? null) + : null; const ctx: UploadContext = { userId: user.id, scoped, @@ -476,7 +463,12 @@ async function processUpload( // count (the unique index has no `deleted_at` predicate), and past the purge // the ledger remembers. if (sourceSystem && sourceId && !queryKey) { - const match = await findSourceKey(user.id, sourceSystem, sourceId); + const match = await findSourceKey( + user.id, + sourceSystem, + sourceId, + sourceInstance, + ); if (match) return answerSourceKey(ctx, match); } @@ -486,157 +478,59 @@ async function processUpload( } catch { return apiError("Failed to read uploaded file", 400); } - if (buffer.byteLength > limits.maxFileBytes) { - return fileTooLarge(limits.maxFileBytes); - } - if (buffer.byteLength === 0) { - return apiError("Uploaded file is empty", 422, { - errorCode: "documents.inbound.invalidMetadata", - }); - } - // §3.1 — magic-byte classification; the wire Content-Type is never trusted. - const detected = detectDocumentType( - buffer, - typeof file.name === "string" ? file.name : null, - ); - if (!detected) { - return apiError("This file type is not supported.", 415, { - errorCode: "documents.inbound.fileType", - reason: "unsupportedType", - }); - } - - // Pre-linking: every episode id must be a LIVE episode of the caller. - const episodeIds = await narrowOwnedEpisodeIds( - user.id, - parsed.data.episodeIds ?? [], - ); - if (episodeIds === null) { - return apiError("Episode not found", 404, { - errorCode: "documents.inbound.episodeNotFound", - }); - } - - // Same for the visit ids the review step offered. A refusal rather than a - // silent drop: the person saw a visit named and would otherwise watch the - // upload succeed without the link they asked for. - const encounterIds = await narrowOwnedEncounterIds( - user.id, - parsed.data.encounterIds ?? [], - ); - if (encounterIds === null) { - return apiError("Visit not found", 404, { - errorCode: "documents.inbound.encounterNotFound", - }); - } - - // sha256 of the PLAINTEXT for same-user duplicate detection. - const contentSha256 = createHash("sha256").update(buffer).digest("hex"); - - // Fast-path dedupe check; the partial unique index closes the race below. - const existing = await prisma.inboundDocument.findFirst({ - where: { userId: user.id, contentSha256, deletedAt: null }, - omit: { contentEncrypted: true }, + // Everything from here to the stored row is the shared ingest path + // (`src/lib/documents/ingest.ts`), the one the document picker's import + // stores through as well. + const result = await ingestDocument({ + userId: user.id, + scoped, + ipAddress: getClientIp(request), + bytes: buffer, + filename: typeof file.name === "string" ? file.name : null, + title: parsed.data.title ?? null, + kind: parsed.data.kind ?? null, + documentDate: parsed.data.documentDate ?? null, + episodeIds: parsed.data.episodeIds ?? [], + encounterIds: parsed.data.encounterIds ?? [], + sourceSystem, + sourceId, + sourceInstance, + aiDeferred: ctx.aiDeferred, + sourceKeyChecked: true, + limits, + // Only the AI work queued after a fresh insert follows the capability; + // the upload itself is data and always accepted. + documentAi: () => getAiCapability("documentAi"), }); - if (existing) { - // An import sending bytes that are already stored under another key (or - // none) is answered with that document — and the key is remembered for - // it, so once the person deletes the document this key stays deleted too. - if (sourceSystem && sourceId) { - const remembered = await rememberSourceAlias( - user.id, - existing.id, - sourceSystem, - sourceId, - ); - if (remembered === "limit") return aliasLimitResponse(); - } - return duplicateResponse(ctx, existing); - } - // Quota gate + insert + pre-links in ONE transaction. Usage counts every - // non-purged row — tombstones still hold TOAST bytes, so "deleted" bytes - // are never invisible weight (undo-delete never changes usage). - let document: SerialisableDocument; - try { - document = await prisma.$transaction(async (tx) => { - // Serialise the quota gate per user: without this, N concurrent - // uploads all read the same SUM before any of them commits and the - // quota can be overshot by up to N × cap in one burst. The advisory - // lock is transaction-scoped (released on commit/rollback) and keyed - // on the user id, so uploads by different users never queue on each - // other. - // (`pg_advisory_xact_lock` returns void, which the client cannot - // deserialize as a column — selecting FROM it yields a plain int row.) - await tx.$queryRaw` - SELECT 1 AS locked - FROM pg_advisory_xact_lock(hashtextextended('documents-quota:' || ${user.id}, 0)) - `; - const rows = await tx.$queryRaw>` - SELECT COALESCE(SUM(byte_size), 0)::bigint AS used - FROM inbound_documents - WHERE user_id = ${user.id} - `; - const usedBytes = Number(rows[0]?.used ?? 0); - if (usedBytes + buffer.byteLength > limits.quotaBytes) { - throw new QuotaExceededError(usedBytes); - } - - const { content, codec } = encryptDocumentContent(buffer); - - // No mass assignment — every column is set field-by-field; `userId` - // comes from the session, never the body. `documentDate` defaults to - // the upload day so display == sort == filter (user-editable later). - const created = await tx.inboundDocument.create({ - data: { - userId: user.id, - kind: parsed.data.kind ?? "OTHER", - // Stored plaintext on purpose (mirrors `filename`) so the list can - // ILIKE-search + ORDER BY it. It MAY hold PHI the user types; that - // is the accepted tradeoff for server-side search/sort — the - // document body stays encrypted. - title: parsed.data.title ?? null, - filename: - typeof file.name === "string" ? file.name.slice(0, 255) : null, - mimeType: detected.mimeType, - byteSize: buffer.byteLength, - contentEncrypted: content, - contentCodec: codec, - contentSha256, - status: "STORED", - documentDate: parsed.data.documentDate - ? isoDateToUtc(parsed.data.documentDate) - : new Date(), - sourceSystem, - sourceId, - aiReadDeferred: ctx.aiDeferred, - }, - omit: { contentEncrypted: true }, + switch (result.kind) { + case "sourceKey": + return answerSourceKey(ctx, result.match); + case "tooLarge": + return fileTooLarge(result.maxFileBytes); + case "empty": + return apiError("Uploaded file is empty", 422, { + errorCode: "documents.inbound.invalidMetadata", }); - - if (episodeIds.length > 0) { - await linkTargets(tx, { - userId: user.id, - sourceKind: "document", - sourceId: created.id, - targetKind: "conditionEpisode", - targetIds: episodeIds, - }); - } - if (encounterIds.length > 0) { - await linkTargets(tx, { - userId: user.id, - sourceKind: "document", - sourceId: created.id, - targetKind: "encounter", - targetIds: encounterIds, - }); - } - return created; - }); - } catch (err) { - if (err instanceof QuotaExceededError) { + case "unsupportedType": + return apiError("This file type is not supported.", 415, { + errorCode: "documents.inbound.fileType", + reason: "unsupportedType", + }); + case "episodeNotFound": + return apiError("Episode not found", 404, { + errorCode: "documents.inbound.episodeNotFound", + }); + case "encounterNotFound": + return apiError("Visit not found", 404, { + errorCode: "documents.inbound.encounterNotFound", + }); + case "aliasLimit": + return aliasLimitResponse(); + case "duplicate": + return duplicateResponse(ctx, result.document); + case "quotaExceeded": // The figures are the owner's; a write-only token learns only that the // vault is full. return apiError("Storage quota exceeded.", 413, { @@ -644,107 +538,13 @@ async function processUpload( reason: "quotaExceeded", ...(scoped ? {} - : { quotaBytes: limits.quotaBytes, usedBytes: err.usedBytes }), + : { quotaBytes: result.quotaBytes, usedBytes: result.usedBytes }), }); - } - if (isP2002(err)) { - // A racing upload won one of the two partial unique indexes — the same - // source key, or the same bytes. Surface the winner exactly as the fast - // paths above would have. - if (sourceSystem && sourceId) { - const match = await findSourceKey(user.id, sourceSystem, sourceId); - if (match) return answerSourceKey(ctx, match); - } - const winner = await prisma.inboundDocument.findFirst({ - where: { userId: user.id, contentSha256, deletedAt: null }, - omit: { contentEncrypted: true }, - }); - if (winner) { - if (sourceSystem && sourceId) { - const remembered = await rememberSourceAlias( - user.id, - winner.id, - sourceSystem, - sourceId, - ); - if (remembered === "limit") return aliasLimitResponse(); - } - return duplicateResponse(ctx, winner); - } - } - throw err; - } - - await auditLog("documents.inbound.store", { - userId: user.id, - ipAddress: getClientIp(request), - details: { - documentId: document.id, - mime: detected.mimeType, - ...(scoped ? { scoped: true } : {}), - ...(sourceSystem ? { sourceSystem } : {}), - }, - }); - - annotate({ - action: { name: "documents.vault.upload" }, - meta: { - documentId: document.id, - byteSize: document.byteSize, - servingClass: detected.servingClass, - linked: episodeIds.length, - linkedVisits: encounterIds.length, - scoped, - sourceSystem, - aiDeferred: ctx.aiDeferred, - }, - }); - - // Auto-index the freshly stored document for content search: enqueue a - // fire-and-forget background job (provider-first, local text-layer fallback). - // The upload response never blocks on or fails because of indexing — the - // enqueue is not awaited and swallows its own errors (a missing boss or a - // transient send failure is a silent no-op). Only fresh inserts enqueue — a - // duplicate upload returns early above and never reaches here. - // - // `aiRead=defer` narrows it to the local text layer: an import of a whole - // archive must not turn into one provider call per file. The person reads - // them later, deliberately, from the document itself. - if (ctx.aiDeferred) { - void enqueueDocumentIndex(user.id, document.id, { localOnly: true }); - } else { - void enqueueDocumentIndex(user.id, document.id); - } - - // Render a preview thumbnail in the background too (pure local compute — no - // egress). Same fire-and-forget contract: the upload never blocks on or fails - // because of it, and a dropped enqueue is recoverable via the boot backfill. - // Only fresh inserts reach here (a duplicate returns early above). - void enqueueDocumentThumbnail(user.id, document.id); - - // Summarise the freshly stored document in the background — but ONLY when the - // `documentsAutoAiRead` opt-in is ON (the job re-checks it and the egress - // consent, and no-ops otherwise) and the `documentAi` capability is open for - // this record. The upload itself never depends on AI: the document is stored - // above whatever the capability says, and the index job still runs its - // provider-free text-layer path. The persisted summary shows on the detail - // view. Same fire-and-forget contract: the upload never blocks on or fails - // because of it. Only fresh inserts reach here (a duplicate returns early). - // - // A deferred upload skips it outright and keeps `summaryState: NONE`, so - // the detail sheet offers "Generate summary" rather than a pending state. - const documentAi = ctx.aiDeferred - ? { available: false as const, reason: "deferred" } - : await getAiCapability("documentAi"); - if (documentAi.available) { - void enqueueDocumentSummary(user.id, document.id); - } else { - annotate({ - action: { name: "documents.summary.enqueueSkipped" }, - meta: { documentId: document.id, reason: documentAi.reason }, - }); + case "stored": + break; } + const document = result.document; if (scoped) return receiptResponse(document.id, 201, { duplicate: false }); const [links, visitLinks] = await Promise.all([ diff --git a/src/app/api/documents/inbound/source/route.ts b/src/app/api/documents/inbound/source/route.ts index caada8b66..ec675a97c 100644 --- a/src/app/api/documents/inbound/source/route.ts +++ b/src/app/api/documents/inbound/source/route.ts @@ -57,6 +57,7 @@ export const GET = apiHandler(async (request: Request) => { const parsed = documentSourceKeySchema.safeParse({ sourceSystem: url.searchParams.get("sourceSystem") ?? undefined, sourceId: url.searchParams.get("sourceId") ?? undefined, + sourceInstance: url.searchParams.get("sourceInstance") ?? undefined, }); if (!parsed.success) { return apiValidationError( @@ -71,6 +72,7 @@ export const GET = apiHandler(async (request: Request) => { user.id, parsed.data.sourceSystem, parsed.data.sourceId, + parsed.data.sourceInstance ?? null, ); annotate({ action: { name: "documents.vault.sourceLookup" }, diff --git a/src/app/api/documents/sources/[system]/import/route.ts b/src/app/api/documents/sources/[system]/import/route.ts new file mode 100644 index 000000000..7da5933af --- /dev/null +++ b/src/app/api/documents/sources/[system]/import/route.ts @@ -0,0 +1,385 @@ +/** + * Document picker (#1038): import one picked document from a connected + * archive into the vault. + * + * One document per request, so the sheet can show each one's progress and + * result, and no single request holds more than one file in memory. The path: + * + * 1. the link target, when the picker was opened from a record, must be one + * of the person's own live records (404 before anything leaves the + * server); + * 2. the source key answers first, without a download: a document already + * imported (or already holding this key as an alias) is a `duplicate`, + * and one the person deleted here is `deleted` — nothing is fetched and + * nothing is stored, exactly as the token upload answers the same key; + * 3. only then is the source asked (on the search allowance, like every + * request to it) and an upload slot taken. One import per key and person + * runs at a time under an advisory lock, so a double click downloads the + * document once. The details and the original file are fetched (bounded + * by the vault's per-file cap); only a file that arrived is charged to + * the vault's own upload allowance (60 an hour, shared with uploads from + * the web and the phone), and it goes through `ingestDocument`, the same + * function the upload route stores through: type check, content dedup, + * quota, provenance (`sourceSystem` / `sourceId` / `sourceInstance`), the + * background jobs; + * 4. the document is linked to the record through `src/lib/links/`. + * + * AI reading follows the person's own setting, unlike the token upload's + * default of holding it back: the person picked this document by hand, as + * deliberately as an upload. The hourly allowance and the daily AI budget + * bound it either way. + * + * Cookie-only and owner-only; see `admitDocumentSourceCaller`. + */ +import { Buffer } from "node:buffer"; + +import { NextRequest, NextResponse } from "next/server"; + +import { apiHandler, requireAuth } from "@/lib/api-handler"; +import { + apiError, + apiValidationError, + getClientIp, + safeJson, + sanitiseZodIssues, +} from "@/lib/api-response"; +import { getAiCapability } from "@/lib/ai/capabilities/gate"; +import { prisma } from "@/lib/db"; +import { + ingestDocument, + personalUploadBucket, + UPLOAD_LIMIT_PER_HOUR, + UPLOAD_WINDOW_MS, +} from "@/lib/documents/ingest"; +import { + checkSourceLookupRateLimit, + findSourceKey, + type SourceKeyMatch, +} from "@/lib/documents/source-key"; +import type { SourceClient } from "@/lib/documents/sources/clients"; +import { clientFor, loadConnection } from "@/lib/documents/sources/connections"; +import { + admitDocumentSourceCaller, + checkSearchRateLimit, + responseForSourceFailure, + sourceErrorResponse, + systemParam, +} from "@/lib/documents/sources/route-support"; +import { + isSourceDocumentId, + type DocumentImportOutcome, + type DocumentPickerLinkKind, + type DocumentPickerSystem, + type DocumentSourceImportDto, +} from "@/lib/documents/sources/types"; +import { + acquireDocumentUploadSlot, + resolveDocumentLimits, +} from "@/lib/documents/upload-policy"; +import { linkTargets } from "@/lib/links"; +import { annotate } from "@/lib/logging/context"; +import { checkRateLimit, rateLimitHeaders } from "@/lib/rate-limit"; +import { documentSourceImportSchema } from "@/lib/validations/document-sources"; +import type { InboundDocumentKindValue } from "@/lib/validations/inbound-documents"; + +export const dynamic = "force-dynamic"; + +type RouteParams = { params: Promise<{ system: string }> }; + +/** How long an import waits for a database connection to hold its lock on. */ +const IMPORT_LOCK_WAIT_MS = 10_000; +/** Longest a single import may hold its key: the download plus the store. */ +const IMPORT_LOCK_HOLD_MS = 150_000; + +interface LinkTarget { + kind: DocumentPickerLinkKind; + id: string; +} + +/** Is the record the picker was opened from one of the person's live ones? */ +async function ownsLinkTarget( + userId: string, + link: LinkTarget, +): Promise { + const where = { id: link.id, userId, deletedAt: null }; + const select = { id: true } as const; + switch (link.kind) { + case "conditionEpisode": + return ( + (await prisma.illnessEpisode.findFirst({ where, select })) !== null + ); + case "encounter": + return (await prisma.encounter.findFirst({ where, select })) !== null; + case "vaccination": + return ( + (await prisma.vaccinationRecord.findFirst({ where, select })) !== null + ); + } +} + +/** File the document against the record, from the record's side. */ +async function linkDocument( + userId: string, + link: LinkTarget | undefined, + documentId: string, +): Promise { + if (!link) return false; + const result = await prisma.$transaction((tx) => + linkTargets(tx, { + userId, + sourceKind: link.kind, + sourceId: link.id, + targetKind: "document", + targetIds: [documentId], + }), + ); + return !result.unknownSource && result.unknownTargetIds.length === 0; +} + +function respond( + system: string, + outcome: DocumentImportOutcome, + documentId: string | null, + linked: boolean, +): NextResponse { + annotate({ + action: { name: "documents.sources.import" }, + meta: { system, outcome, linked, documentId }, + }); + const data: DocumentSourceImportDto = { outcome, documentId, linked }; + return NextResponse.json( + { data, error: null }, + { status: outcome === "imported" ? 201 : 200 }, + ); +} + +/** A key the vault already holds: link a live one, never revive a deleted one. */ +async function answerHeldKey( + system: string, + userId: string, + link: LinkTarget | undefined, + match: SourceKeyMatch, +): Promise { + if (match.state === "deleted") { + return respond(system, "deleted", match.id, false); + } + const linked = await linkDocument(userId, link, match.document.id); + return respond(system, "duplicate", match.document.id, linked); +} + +export const POST = apiHandler( + async (request: NextRequest, { params }: RouteParams) => { + const auth = await requireAuth(); + const refused = await admitDocumentSourceCaller(auth); + if (refused) return refused; + const system = await systemParam(params); + if (system instanceof Response) return system; + const userId = auth.user.id; + + const { data: body, error: jsonError } = await safeJson(request, { + maxBytes: 4 * 1024, + }); + if (jsonError) return jsonError; + const parsed = documentSourceImportSchema.safeParse(body); + if (!parsed.success) { + return apiValidationError( + "Invalid import request", + sanitiseZodIssues(parsed.error.issues), + 422, + ); + } + const { sourceId, kind, link } = parsed.data; + // Only an id this system could have issued goes into a request path. + if (!isSourceDocumentId(system, sourceId)) { + return apiError("Not a document id of this archive.", 422); + } + + const row = await loadConnection(userId, system); + if (!row) return sourceErrorResponse("notConnected"); + let client; + try { + client = clientFor(row); + } catch (err) { + return responseForSourceFailure(err); + } + const instance = client.instance; + + if (link && !(await ownsLinkTarget(userId, link))) { + return sourceErrorResponse("linkTargetNotFound"); + } + + // The source key answers before anything is downloaded or charged, on the + // same metered lookup allowance the upload's key check draws on. + const lookupRl = await checkSourceLookupRateLimit( + false, + auth.session.id, + userId, + ); + if (!lookupRl.allowed) { + return sourceErrorResponse( + "rateLimited", + undefined, + rateLimitHeaders(lookupRl), + ); + } + const held = await findSourceKey(userId, system, sourceId, instance); + if (held) return answerHeldKey(system, userId, link, held); + + // A request to the source counts against the search allowance like any + // other; the upload allowance is charged only once a file arrived. + const fetchRl = await checkSearchRateLimit(userId); + if (!fetchRl.allowed) { + return sourceErrorResponse( + "rateLimited", + undefined, + rateLimitHeaders(fetchRl), + ); + } + + const release = acquireDocumentUploadSlot(userId); + if (!release) { + return apiError( + "Too many uploads are already in progress. Try again shortly.", + 429, + { + errorCode: "documents.inbound.uploadBusy", + reason: "uploadBusy", + headers: { "Retry-After": "1" }, + }, + ); + } + + try { + // One import of one key at a time per person: a second tab or a double + // click waits here, then finds the key the first one stored, so the + // document is downloaded once. The lock is transaction-scoped and the + // transaction holds nothing else; the store runs in its own. + return await prisma.$transaction( + async (tx) => { + await tx.$queryRaw` + SELECT 1 AS locked + FROM pg_advisory_xact_lock(hashtextextended(${`documents-import:${userId}:${system}:${instance}:${sourceId}`}, 0)) + `; + const raced = await findSourceKey(userId, system, sourceId, instance); + if (raced) return answerHeldKey(system, userId, link, raced); + return importOne({ + request, + client, + userId, + system, + instance, + sourceId, + kind: kind ?? null, + link, + }); + }, + { maxWait: IMPORT_LOCK_WAIT_MS, timeout: IMPORT_LOCK_HOLD_MS }, + ); + } finally { + release(); + } + }, +); + +/** Fetch one document and store it; the caller holds the key's lock. */ +async function importOne(args: { + request: Request; + client: SourceClient; + userId: string; + system: DocumentPickerSystem; + instance: string; + sourceId: string; + kind: InboundDocumentKindValue | null; + link: LinkTarget | undefined; +}): Promise { + const { request, client, userId, system, instance, sourceId, kind, link } = + args; + const limits = await resolveDocumentLimits(userId); + let meta; + let file; + try { + meta = await client.document(sourceId); + file = await client.download(sourceId, limits.maxFileBytes); + } catch (err) { + return responseForSourceFailure(err); + } + + const rl = await checkRateLimit( + personalUploadBucket(userId), + UPLOAD_LIMIT_PER_HOUR, + UPLOAD_WINDOW_MS, + ); + if (!rl.allowed) { + return apiError("Too many uploads. Try again later.", 429, { + errorCode: "documents.inbound.rateLimited", + headers: rateLimitHeaders(rl), + }); + } + + const result = await ingestDocument({ + userId, + scoped: false, + ipAddress: getClientIp(request), + bytes: Buffer.from(file.bytes), + filename: meta.filename ?? file.filename, + title: meta.title, + kind, + documentDate: meta.date, + episodeIds: [], + encounterIds: [], + sourceSystem: system, + sourceId, + sourceInstance: instance, + aiDeferred: false, + sourceKeyChecked: true, + limits, + // The import is data; only the summary queued after a fresh insert + // follows the capability (and the person's auto-read setting). + documentAi: () => getAiCapability("documentAi"), + }); + + switch (result.kind) { + case "sourceKey": + return answerHeldKey(system, userId, link, result.match); + case "duplicate": { + // The same bytes were already in the vault (an upload, or another + // archive); the key is now remembered for that document. + const linked = await linkDocument(userId, link, result.document.id); + return respond(system, "duplicate", result.document.id, linked); + } + case "stored": { + const linked = await linkDocument(userId, link, result.document.id); + return respond(system, "imported", result.document.id, linked); + } + case "tooLarge": + return apiError("File is too large.", 413, { + errorCode: "documents.inbound.fileTooLarge", + reason: "fileTooLarge", + maxFileBytes: result.maxFileBytes, + }); + case "empty": + case "unsupportedType": + return apiError("This file type is not supported.", 415, { + errorCode: "documents.inbound.fileType", + reason: "unsupportedType", + }); + case "quotaExceeded": + return apiError("Storage quota exceeded.", 413, { + errorCode: "documents.inbound.quotaExceeded", + reason: "quotaExceeded", + quotaBytes: result.quotaBytes, + usedBytes: result.usedBytes, + }); + case "aliasLimit": + return apiError( + "This file is already stored under too many source ids.", + 409, + { errorCode: "documents.inbound.sourceAliasLimit" }, + ); + case "episodeNotFound": + case "encounterNotFound": + // No pre-links are passed on this path; unreachable, but a result + // the switch does not name must not fall through silently. + return sourceErrorResponse("linkTargetNotFound"); + } +} diff --git a/src/app/api/documents/sources/[system]/route.ts b/src/app/api/documents/sources/[system]/route.ts new file mode 100644 index 000000000..91cfb0dba --- /dev/null +++ b/src/app/api/documents/sources/[system]/route.ts @@ -0,0 +1,204 @@ +/** + * Document picker (#1038): save or remove the person's connection to one + * document archive (`paperless` or `papra`). + * + * PUT validates the base address against the operator's list + * (`DOCUMENT_SOURCE_ORIGINS`), tests the connection live before storing + * anything (a wrong token or an unreachable host is reported here, not on the + * first search), then stores the token encrypted. A PUT without a token keeps + * the stored one, so the base path or the Papra organization can change + * without typing the key again, but only on the same origin: a new origin, and + * the first save, must carry it (422 `documents.sources.tokenRequired`). The response is the + * connection without its token, like every read of it. + * + * DELETE removes the row, and with it the token. + * + * Cookie-only and owner-only; see `admitDocumentSourceCaller`. + */ +import { NextRequest } from "next/server"; + +import { apiHandler, requireAuth } from "@/lib/api-handler"; +import { + apiError, + apiSuccess, + apiValidationError, + getClientIp, + safeJson, + sanitiseZodIssues, +} from "@/lib/api-response"; +import { auditLog } from "@/lib/auth/audit"; +import { decrypt, encrypt } from "@/lib/crypto"; +import { prisma } from "@/lib/db"; +import { sourceClient } from "@/lib/documents/sources/clients"; +import { + loadConnection, + toConnectionDto, +} from "@/lib/documents/sources/connections"; +import { DocumentSourceError } from "@/lib/documents/sources/http"; +import { evaluateSourceBaseUrl } from "@/lib/documents/sources/origins"; +import { + admitDocumentSourceCaller, + checkConnectRateLimit, + responseForSourceFailure, + sourceErrorResponse, + systemParam, +} from "@/lib/documents/sources/route-support"; +import { annotate } from "@/lib/logging/context"; +import { rateLimitHeaders } from "@/lib/rate-limit"; +import { documentSourceSaveSchema } from "@/lib/validations/document-sources"; + +export const dynamic = "force-dynamic"; + +type RouteParams = { params: Promise<{ system: string }> }; + +export const PUT = apiHandler( + async (request: NextRequest, { params }: RouteParams) => { + const auth = await requireAuth(); + const refused = await admitDocumentSourceCaller(auth); + if (refused) return refused; + const system = await systemParam(params); + if (system instanceof Response) return system; + const userId = auth.user.id; + annotate({ action: { name: "documents.sources.save" }, meta: { system } }); + + const rl = await checkConnectRateLimit(userId); + if (!rl.allowed) { + return sourceErrorResponse( + "rateLimited", + undefined, + rateLimitHeaders(rl), + ); + } + + const { data: body, error: jsonError } = await safeJson(request, { + maxBytes: 8 * 1024, + }); + if (jsonError) return jsonError; + const parsed = documentSourceSaveSchema.safeParse(body); + if (!parsed.success) { + return apiValidationError( + "Invalid connection details", + sanitiseZodIssues(parsed.error.issues), + 422, + ); + } + + const verdict = evaluateSourceBaseUrl(parsed.data.baseUrl); + if (!verdict.ok) { + return verdict.reason === "notAllowed" + ? sourceErrorResponse("originNotAllowed") + : apiError("Enter the full address, starting with https://.", 422, { + errorCode: "documents.sources.invalidAddress", + }); + } + + const organizationId = + system === "PAPRA" ? (parsed.data.organizationId ?? null) : null; + if (system === "PAPRA" && !organizationId) { + return apiError("Papra needs the organization id.", 422, { + errorCode: "documents.sources.organizationRequired", + }); + } + + const existing = await loadConnection(userId, system); + // The stored token is only ever sent back to the origin it was saved + // for. A new address on another origin needs the token typed again, or + // saving an address would hand the old archive's token to whatever the + // new one is. + const reusable = + existing !== null && + evaluateSourceBaseUrl(existing.baseUrl, new Set([verdict.origin])).ok; + const token = + parsed.data.token ?? + (existing && reusable ? decrypt(existing.tokenEncrypted) : null); + if (!token) { + return apiError("Enter the API token.", 422, { + errorCode: "documents.sources.tokenRequired", + }); + } + + // Live check before anything is stored: the row only ever holds a + // connection that answered once. + try { + await sourceClient({ + system, + origin: verdict.origin, + baseUrl: verdict.baseUrl, + organizationId, + token, + }).test(); + } catch (err) { + if (err instanceof DocumentSourceError) { + annotate({ meta: { refused: err.code } }); + } + return responseForSourceFailure(err); + } + + const now = new Date(); + const tokenEncrypted = encrypt(token); + // Field by field; `userId` from the session, never the body. + const saved = await prisma.documentSourceConnection.upsert({ + where: { userId_system: { userId, system } }, + create: { + userId, + system, + baseUrl: verdict.baseUrl, + organizationId, + tokenEncrypted, + lastVerifiedAt: now, + }, + update: { + baseUrl: verdict.baseUrl, + organizationId, + tokenEncrypted, + lastVerifiedAt: now, + }, + select: { + system: true, + baseUrl: true, + organizationId: true, + lastVerifiedAt: true, + }, + }); + + await auditLog("documents.sources.connect", { + userId, + ipAddress: getClientIp(request), + details: { system, origin: verdict.origin, replaced: existing !== null }, + }); + + return apiSuccess(toConnectionDto(saved)); + }, +); + +export const DELETE = apiHandler( + async (request: NextRequest, { params }: RouteParams) => { + const auth = await requireAuth(); + // Neither the list nor the module is needed to forget a connection: an + // operator who removed the origin, or a person who switched Documents + // off, must still be able to delete the stored token. + const refused = await admitDocumentSourceCaller(auth, { + needsList: false, + needsModule: false, + }); + if (refused) return refused; + const system = await systemParam(params); + if (system instanceof Response) return system; + annotate({ + action: { name: "documents.sources.disconnect" }, + meta: { system }, + }); + + const { count } = await prisma.documentSourceConnection.deleteMany({ + where: { userId: auth.user.id, system }, + }); + if (count > 0) { + await auditLog("documents.sources.disconnect", { + userId: auth.user.id, + ipAddress: getClientIp(request), + details: { system }, + }); + } + return apiSuccess({ disconnected: count > 0 }); + }, +); diff --git a/src/app/api/documents/sources/[system]/search/route.ts b/src/app/api/documents/sources/[system]/search/route.ts new file mode 100644 index 000000000..edaf7a9c3 --- /dev/null +++ b/src/app/api/documents/sources/[system]/search/route.ts @@ -0,0 +1,159 @@ +/** + * Document picker (#1038): search one connected document archive. + * + * `q` searches the document's name (the title in Paperless-ngx, the name in + * Papra), `tag` narrows to one of the source's tags by its id, `from` / `to` + * to a date range (inclusive), `page` pages through 25 at a time. The answer + * is a minimal row per document (source id, title, date, tag names, size when + * the source says it) plus what HealthLog already holds under that source key: + * `new`, `imported` (with the vault document's id) or `deleted` (the person + * deleted it here; importing it again is refused). Nothing is stored. + * + * When the source ignores a filter and the local re-check empties a page, the + * route reads on (up to five source pages) and answers with the last page it + * read, so the client's "load more" continues after it. + * + * Sixty searches a minute, shared with the tag read. Cookie-only and + * owner-only; see `admitDocumentSourceCaller`. + */ +import { NextRequest } from "next/server"; + +import { apiHandler, requireAuth } from "@/lib/api-handler"; +import { + apiSuccess, + apiValidationError, + sanitiseZodIssues, +} from "@/lib/api-response"; +import { findSourceKeyStates } from "@/lib/documents/source-key"; +import type { SourceListItem } from "@/lib/documents/sources/clients"; +import { openConnection } from "@/lib/documents/sources/connections"; +import { + admitDocumentSourceCaller, + checkSearchRateLimit, + responseForSourceFailure, + sourceErrorResponse, + systemParam, +} from "@/lib/documents/sources/route-support"; +import { + DOCUMENT_PICKER_MAX_PAGE, + type DocumentSourceResultDto, + type DocumentSourceSearchDto, +} from "@/lib/documents/sources/types"; +import { annotate } from "@/lib/logging/context"; +import { rateLimitHeaders } from "@/lib/rate-limit"; +import { documentSourceSearchSchema } from "@/lib/validations/document-sources"; + +export const dynamic = "force-dynamic"; + +/** Source pages one search request may read to find a non-empty one. */ +const SEARCH_PAGES_PER_REQUEST = 5; + +type RouteParams = { params: Promise<{ system: string }> }; + +export const GET = apiHandler( + async (request: NextRequest, { params }: RouteParams) => { + const auth = await requireAuth(); + const refused = await admitDocumentSourceCaller(auth); + if (refused) return refused; + const system = await systemParam(params); + if (system instanceof Response) return system; + const userId = auth.user.id; + + const url = new URL(request.url); + const parsed = documentSourceSearchSchema.safeParse({ + q: url.searchParams.get("q") ?? undefined, + tag: url.searchParams.get("tag") ?? undefined, + from: url.searchParams.get("from") ?? undefined, + to: url.searchParams.get("to") ?? undefined, + page: url.searchParams.get("page") ?? undefined, + }); + if (!parsed.success) { + return apiValidationError( + "Invalid search", + sanitiseZodIssues(parsed.error.issues), + 422, + ); + } + const { q, tag, from, to, page } = parsed.data; + + const rl = await checkSearchRateLimit(userId); + if (!rl.allowed) { + return sourceErrorResponse( + "rateLimited", + undefined, + rateLimitHeaders(rl), + ); + } + + // A page the local tag/date re-check emptied is not an answer: the next + // source page may hold matches. Read on, a bounded number of pages, and + // report the last one read so "load more" continues after it. + let found: { items: SourceListItem[]; hasMore: boolean } = { + items: [], + hasMore: false, + }; + let lastPage = page; + let instance: string; + try { + const client = await openConnection(userId, system); + instance = client.instance; + for ( + let next = page; + next < page + SEARCH_PAGES_PER_REQUEST && + next <= DOCUMENT_PICKER_MAX_PAGE; + next += 1 + ) { + found = await client.search({ + q, + tagId: tag ?? null, + from: from ?? null, + to: to ?? null, + page: next, + }); + lastPage = next; + if (found.items.length > 0 || !found.hasMore) break; + } + } catch (err) { + return responseForSourceFailure(err); + } + + const states = await findSourceKeyStates( + userId, + system, + found.items.map((item) => item.sourceId), + instance, + ); + const results: DocumentSourceResultDto[] = found.items.map((item) => { + const held = states.get(item.sourceId); + return { + sourceId: item.sourceId, + title: item.title, + date: item.date, + tags: item.tags, + sizeBytes: item.sizeBytes, + state: held?.state ?? "new", + documentId: held?.state === "imported" ? held.documentId : null, + }; + }); + + // Counts only: the query and the titles are the person's own words and + // stay out of the wide event. + annotate({ + action: { name: "documents.sources.search" }, + meta: { + system, + page, + pagesRead: lastPage - page + 1, + count: results.length, + filtered: Boolean(q || tag || from || to), + }, + }); + + const body: DocumentSourceSearchDto = { + results, + page: lastPage, + hasMore: found.hasMore && lastPage < DOCUMENT_PICKER_MAX_PAGE, + }; + return apiSuccess(body); + }, +); diff --git a/src/app/api/documents/sources/[system]/tags/route.ts b/src/app/api/documents/sources/[system]/tags/route.ts new file mode 100644 index 000000000..4b788608c --- /dev/null +++ b/src/app/api/documents/sources/[system]/tags/route.ts @@ -0,0 +1,52 @@ +/** + * Document picker (#1038): the tags of one connected document archive, for the + * picker's tag filter. The first 250 by name, as `{ id, name }` in the + * source's own ids. Shares the search allowance (60 a minute). + * + * Cookie-only and owner-only; see `admitDocumentSourceCaller`. + */ +import { NextRequest } from "next/server"; + +import { apiHandler, requireAuth } from "@/lib/api-handler"; +import { apiSuccess } from "@/lib/api-response"; +import { openConnection } from "@/lib/documents/sources/connections"; +import { + admitDocumentSourceCaller, + checkSearchRateLimit, + responseForSourceFailure, + sourceErrorResponse, + systemParam, +} from "@/lib/documents/sources/route-support"; +import { annotate } from "@/lib/logging/context"; +import { rateLimitHeaders } from "@/lib/rate-limit"; + +export const dynamic = "force-dynamic"; + +type RouteParams = { params: Promise<{ system: string }> }; + +export const GET = apiHandler( + async (_request: NextRequest, { params }: RouteParams) => { + const auth = await requireAuth(); + const refused = await admitDocumentSourceCaller(auth); + if (refused) return refused; + const system = await systemParam(params); + if (system instanceof Response) return system; + annotate({ action: { name: "documents.sources.tags" }, meta: { system } }); + + const rl = await checkSearchRateLimit(auth.user.id); + if (!rl.allowed) { + return sourceErrorResponse( + "rateLimited", + undefined, + rateLimitHeaders(rl), + ); + } + + try { + const client = await openConnection(auth.user.id, system); + return apiSuccess({ tags: await client.tags() }); + } catch (err) { + return responseForSourceFailure(err); + } + }, +); diff --git a/src/app/api/documents/sources/[system]/test/route.ts b/src/app/api/documents/sources/[system]/test/route.ts new file mode 100644 index 000000000..536b0c7d1 --- /dev/null +++ b/src/app/api/documents/sources/[system]/test/route.ts @@ -0,0 +1,71 @@ +/** + * Document picker (#1038): test the saved connection to one document archive. + * + * One request to the source with the stored token, answered `{ ok, latencyMs }` + * or with the reason it failed (`meta.errorCode`, and `meta.upstreamStatus` + * when the source answered at all; its body is never passed on). A success + * stamps `lastVerifiedAt`. Shares the save's allowance of ten a minute. + * + * Cookie-only and owner-only; see `admitDocumentSourceCaller`. + */ +import { NextRequest } from "next/server"; + +import { apiHandler, requireAuth } from "@/lib/api-handler"; +import { apiSuccess } from "@/lib/api-response"; +import { + markVerified, + openConnection, +} from "@/lib/documents/sources/connections"; +import { DocumentSourceError } from "@/lib/documents/sources/http"; +import { + admitDocumentSourceCaller, + checkConnectRateLimit, + responseForSourceFailure, + sourceErrorResponse, + systemParam, +} from "@/lib/documents/sources/route-support"; +import { annotate } from "@/lib/logging/context"; +import { rateLimitHeaders } from "@/lib/rate-limit"; + +export const dynamic = "force-dynamic"; + +type RouteParams = { params: Promise<{ system: string }> }; + +export const POST = apiHandler( + async (_request: NextRequest, { params }: RouteParams) => { + const auth = await requireAuth(); + const refused = await admitDocumentSourceCaller(auth); + if (refused) return refused; + const system = await systemParam(params); + if (system instanceof Response) return system; + const userId = auth.user.id; + annotate({ action: { name: "documents.sources.test" }, meta: { system } }); + + const rl = await checkConnectRateLimit(userId); + if (!rl.allowed) { + return sourceErrorResponse( + "rateLimited", + undefined, + rateLimitHeaders(rl), + ); + } + + const start = performance.now(); + try { + const client = await openConnection(userId, system); + await client.test(); + } catch (err) { + if (err instanceof DocumentSourceError) { + annotate({ meta: { refused: err.code } }); + } + return responseForSourceFailure(err); + } + const latencyMs = Math.round(performance.now() - start); + const verifiedAt = await markVerified(userId, system); + return apiSuccess({ + ok: true, + latencyMs, + lastVerifiedAt: verifiedAt.toISOString(), + }); + }, +); diff --git a/src/app/api/documents/sources/route.ts b/src/app/api/documents/sources/route.ts new file mode 100644 index 000000000..e79f4b22f --- /dev/null +++ b/src/app/api/documents/sources/route.ts @@ -0,0 +1,61 @@ +/** + * Document picker (#1038): which document archives this person can pick from. + * + * `available` says whether the operator listed any origin in + * `DOCUMENT_SOURCE_ORIGINS`; `allowedOrigins` lists them so the settings card + * can say which addresses will be accepted; `connections` are the person's own + * saved connections, never with their token (`hasToken` is the literal true). + * No request leaves the server here. + * + * Cookie-only and owner-only like every route under this path + * (`admitDocumentSourceCaller`), but it needs neither the documents module nor + * the list: with the list unset it answers 200 `available: false` (plus any + * rows still stored, so they can be deleted) rather than 404, because this is + * the one read the client makes to decide what to show. + */ +import { apiHandler, requireAuth } from "@/lib/api-handler"; +import { apiSuccess } from "@/lib/api-response"; +import { + listConnections, + toConnectionDto, +} from "@/lib/documents/sources/connections"; +import { + documentSourcesAvailable, + listedDocumentSourceOrigins, +} from "@/lib/documents/sources/origins"; +import type { DocumentSourcesStatusDto } from "@/lib/documents/sources/types"; +import { admitDocumentSourceCaller } from "@/lib/documents/sources/route-support"; +import { annotate } from "@/lib/logging/context"; + +export const dynamic = "force-dynamic"; + +export const GET = apiHandler(async () => { + const auth = await requireAuth(); + const refused = await admitDocumentSourceCaller(auth, { + needsList: false, + needsModule: false, + }); + if (refused) return refused; + + annotate({ action: { name: "documents.sources.status" } }); + + // Rows are listed even with the picker off: a connection whose origin the + // operator removed (or whose list is gone) must stay visible so it can be + // deleted. `originAllowed` is false for each of them then. + const rows = await listConnections(auth.user.id); + if (!documentSourcesAvailable()) { + const off: DocumentSourcesStatusDto = { + available: false, + allowedOrigins: [], + connections: rows.map(toConnectionDto), + }; + return apiSuccess(off); + } + + const status: DocumentSourcesStatusDto = { + available: true, + allowedOrigins: listedDocumentSourceOrigins(), + connections: rows.map(toConnectionDto), + }; + return apiSuccess(status); +}); diff --git a/src/components/documents/documents-view.tsx b/src/components/documents/documents-view.tsx index 67e5ac4aa..c68bb8f5f 100644 --- a/src/components/documents/documents-view.tsx +++ b/src/components/documents/documents-view.tsx @@ -67,6 +67,7 @@ import { DocumentShareSheet } from "./document-share-sheet"; import { DocumentFilterBar, type ConditionChip } from "./document-filter-bar"; import { DocumentTimeline } from "./document-timeline"; import { UploadZone } from "./upload-zone"; +import { ImportFromSourceButton } from "./sources/import-from-source-button"; import { useReindexAll } from "./use-content-index"; import { useDocumentUpload } from "./use-document-upload"; import { usePageFileDrop } from "./use-page-file-drop"; @@ -876,13 +877,17 @@ export function DocumentsView() { description={t("documents.subtitle")} actions={ canManageDocuments ? ( - +

+ {/* Renders only with a connected Paperless-ngx or Papra. */} + + +
) : null } /> diff --git a/src/components/documents/episode-documents-card.tsx b/src/components/documents/episode-documents-card.tsx index fc42f410b..2c9bc580f 100644 --- a/src/components/documents/episode-documents-card.tsx +++ b/src/components/documents/episode-documents-card.tsx @@ -32,6 +32,7 @@ import { useFormatters, useTranslations } from "@/lib/i18n/context"; import { queryKeys } from "@/lib/query-keys"; import type { InboundDocumentDto } from "@/lib/validations/inbound-documents"; import { DocumentLinkPicker } from "./document-link-picker"; +import { ImportFromSourceButton } from "./sources/import-from-source-button"; import { DOCUMENT_KIND_ICONS } from "./document-kind-meta"; import { documentDateKey } from "./vault-utils"; @@ -167,6 +168,9 @@ export function EpisodeDocumentsCard({ episodeId }: { episodeId: string }) { {t("documents.episodeCard.upload")} + ) : null} {hasMore ? ( diff --git a/src/components/documents/sources/__tests__/document-sources-ui.test.tsx b/src/components/documents/sources/__tests__/document-sources-ui.test.tsx new file mode 100644 index 000000000..208163bcb --- /dev/null +++ b/src/components/documents/sources/__tests__/document-sources-ui.test.tsx @@ -0,0 +1,275 @@ +/** + * The document picker's client gates (#1038), pinned via static renders: + * + * - the import button exists only for a person in their own record with a + * usable connection; a delegate, an unset operator list, no connection, or + * a connection whose origin was removed leave no button; + * - its label names the one connected system, or stays generic for two; + * - the settings card renders nothing unless the operator enabled the + * picker, lists the allowed origins, and never puts a token in the markup; + * - every error code the routes emit has its own sentence, and an unknown + * one falls back to the generic sentence rather than a raw key. + */ +import { describe, expect, it, vi } from "vitest"; +import { renderToStaticMarkup } from "react-dom/server"; +import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; + +let modules: Record = { inboundDocuments: true }; +vi.mock("@/hooks/use-auth", () => ({ + useAuth: () => ({ + user: { id: "u1", modules }, + isLoading: false, + isAuthenticated: true, + }), +})); + +let shared = false; +vi.mock("@/hooks/use-record-capabilities", () => ({ + useRecordCapabilities: () => ({ + inSharedRecord: shared, + canManageDomain: () => !shared, + }), +})); + +vi.mock("@/lib/api/api-fetch", async () => { + const actual = await vi.importActual( + "@/lib/api/api-fetch", + ); + return { ...actual, apiGet: () => new Promise(() => {}) }; +}); + +import { DocumentSourcesCard } from "@/components/settings/integrations/document-sources-card"; +import en from "../../../../../messages/en.json"; +import { ApiError } from "@/lib/api/api-fetch"; +import { + DOCUMENT_SOURCE_ERROR_CODES, + type DocumentSourcesStatusDto, +} from "@/lib/documents/sources/types"; +import { I18nProvider } from "@/lib/i18n/context"; +import { queryKeys } from "@/lib/query-keys"; + +import { ImportFromSourceButton } from "../import-from-source-button"; +import { mergeIds } from "../merge-ids"; +import { nextChoiceIndex } from "../segmented-choice"; +import { errorCodeOf, sourceErrorMessage } from "../source-errors"; + +const PAPERLESS = { + system: "PAPERLESS" as const, + baseUrl: "http://paperless.lan:8000", + organizationId: null, + hasToken: true as const, + lastVerifiedAt: "2026-09-20T10:00:00.000Z", + originAllowed: true, +}; +const PAPRA = { + ...PAPERLESS, + system: "PAPRA" as const, + baseUrl: "https://papra.example.com", + organizationId: "org_1", +}; + +function render(node: React.ReactNode, status?: DocumentSourcesStatusDto) { + const queryClient = new QueryClient({ + defaultOptions: { queries: { retry: false } }, + }); + if (status) + queryClient.setQueryData(queryKeys.documentSourcesStatus(), status); + return renderToStaticMarkup( + + {node} + , + ); +} + +const on = (connections: DocumentSourcesStatusDto["connections"]) => ({ + available: true, + allowedOrigins: ["http://paperless.lan:8000", "https://papra.example.com"], + connections, +}); + +describe("", () => { + it("names the one connected system", () => { + const html = render(, on([PAPERLESS])); + expect(html).toContain('data-slot="document-source-open"'); + expect(html).toContain("Import from Paperless-ngx"); + }); + + it("stays generic with two", () => { + const html = render(, on([PAPERLESS, PAPRA])); + expect(html).toContain("Import from archive"); + }); + + it("renders nothing without a usable connection", () => { + expect(render(, on([]))).toBe(""); + expect( + render( + , + on([{ ...PAPERLESS, originAllowed: false }]), + ), + ).toBe(""); + expect( + render(, { + available: false, + allowedOrigins: [], + connections: [], + }), + ).toBe(""); + }); + + it("renders nothing for a delegate, even with the owner's data cached", () => { + shared = true; + try { + expect(render(, on([PAPERLESS]))).toBe(""); + } finally { + shared = false; + } + }); + + it("renders nothing with the documents module off", () => { + modules = { inboundDocuments: false }; + try { + expect(render(, on([PAPERLESS]))).toBe(""); + } finally { + modules = { inboundDocuments: true }; + } + }); +}); + +describe("", () => { + it("renders nothing when the operator has not enabled the picker", () => { + expect( + render(, { + available: false, + allowedOrigins: [], + connections: [], + }), + ).toBe(""); + expect(render()).toBe(""); + }); + + it("renders nothing for a delegate", () => { + shared = true; + try { + expect(render(, on([PAPERLESS]))).toBe(""); + } finally { + shared = false; + } + }); + + it("shows the form, the allowed origins and a saved-token placeholder, never a token", () => { + const html = render(, on([PAPERLESS])); + expect(html).toContain('data-testid="document-sources-card"'); + expect(html).toContain("Document archives"); + expect(html).toContain( + "Allowed on this server: http://paperless.lan:8000, https://papra.example.com", + ); + expect(html).toContain("Saved for this address. Leave empty to keep it."); + expect(html).toContain('value="http://paperless.lan:8000"'); + // The password field is empty: the server never sends the token back. + expect(html).toMatch(/id="document-source-token-paperless"[^>]*value=""/); + expect(html).toContain("Disconnect"); + }); + + it("keeps a stored connection visible and removable once the operator list is gone", () => { + const html = render(, { + available: false, + allowedOrigins: [], + connections: [{ ...PAPERLESS, originAllowed: false }], + }); + expect(html).toContain('data-testid="document-sources-card"'); + expect(html).toContain("no longer allows document archives"); + expect(html).toContain("Disconnect"); + expect(html).not.toContain("Save and test"); + expect(html).not.toContain('id="document-source-token-paperless"'); + }); + + it("keeps a stored connection removable with the documents module off", () => { + modules = { inboundDocuments: false }; + try { + const html = render(, on([PAPERLESS])); + expect(html).toContain("Documents are switched off"); + expect(html).toContain("Disconnect"); + expect(html).not.toContain("Save and test"); + } finally { + modules = { inboundDocuments: true }; + } + }); + + it("uses one tab stop for the system switch", () => { + const html = render(, on([PAPERLESS])); + expect(html.match(/role="radio"[^>]*tabindex="0"/gi)?.length).toBe(1); + expect(html.match(/role="radio"[^>]*tabindex="-1"/gi)?.length).toBe(1); + }); + + it("offers no disconnect or test for a system that is not connected", () => { + const html = render(, on([])); + expect(html).toContain("Not connected"); + expect(html).not.toContain("Disconnect"); + expect(html).toContain("Save and test"); + }); +}); + +describe("source error sentences", () => { + const t = (key: string) => { + let node: unknown = en; + for (const part of key.split(".")) { + node = (node as Record | undefined)?.[part]; + } + return typeof node === "string" ? node : key; + }; + const generic = t("documents.sourcePicker.errors.generic"); + + it("has its own sentence for every code the routes emit", () => { + const codes = [ + ...DOCUMENT_SOURCE_ERROR_CODES, + "documents.sources.browserOnly", + "documents.sources.invalidAddress", + "documents.sources.organizationRequired", + "documents.sources.tokenRequired", + "documents.inbound.fileTooLarge", + "documents.inbound.quotaExceeded", + "documents.inbound.fileType", + "documents.inbound.rateLimited", + "documents.inbound.uploadBusy", + "module.disabled", + ]; + const sentences = new Set(); + for (const code of codes) { + const sentence = sourceErrorMessage(t, code); + expect(sentence, code).not.toBe(generic); + expect(sentence, code).not.toContain("documents.sourcePicker"); + sentences.add(sentence); + } + expect(sentences.size).toBe(codes.length); + expect(sourceErrorMessage(t, "something.else")).toBe(generic); + expect(sourceErrorMessage(t, undefined)).toBe(generic); + }); + + it("reads the code off a failed request", () => { + expect( + errorCodeOf( + new ApiError("x", 502, { errorCode: "documents.sources.authRefused" }), + ), + ).toBe("documents.sources.authRefused"); + expect(errorCodeOf(new Error("x"))).toBeUndefined(); + }); +}); + +describe("nextChoiceIndex", () => { + it("moves with the arrows, wraps, and jumps with Home and End", () => { + expect(nextChoiceIndex("ArrowRight", 0, 2)).toBe(1); + expect(nextChoiceIndex("ArrowRight", 1, 2)).toBe(0); + expect(nextChoiceIndex("ArrowLeft", 0, 2)).toBe(1); + expect(nextChoiceIndex("ArrowDown", 0, 3)).toBe(1); + expect(nextChoiceIndex("ArrowUp", 0, 3)).toBe(2); + expect(nextChoiceIndex("Home", 2, 3)).toBe(0); + expect(nextChoiceIndex("End", 0, 3)).toBe(2); + expect(nextChoiceIndex("Enter", 0, 3)).toBeNull(); + }); +}); + +describe("mergeIds", () => { + it("adds imported ids to a form's selection once, in order", () => { + expect(mergeIds(["a", "b"], ["b", "c", "c"])).toEqual(["a", "b", "c"]); + }); +}); diff --git a/src/components/documents/sources/document-source-picker.tsx b/src/components/documents/sources/document-source-picker.tsx new file mode 100644 index 000000000..09dc62404 --- /dev/null +++ b/src/components/documents/sources/document-source-picker.tsx @@ -0,0 +1,764 @@ +"use client"; + +/** + * The document picker (#1038): search a connected Paperless-ngx or Papra by + * document name, narrow by tag and date, tick the documents to bring over, + * import them. + * + * Name first, because that is how the person looks for a letter: the search + * field opens focused and debounces into the source's own name search. Tag and + * date are filters beside it, not the entry point. Each result says what + * HealthLog already holds under that source key: already imported, deleted + * here (not offered again: a deletion was a decision), or new. + * + * The import runs one document per request, in order, so every row can show + * its own progress and result, and one failure does not hide the others. A + * refusal that will answer every following document the same way (the hourly + * upload allowance, a full vault, a refused token) stops the run and says so. + * + * Where it was opened from decides what an import does besides storing: + * + * - `link`: the record already exists (a condition's page), so the server + * files the document against it as part of the import; + * - `onImported`: a form that owns its selection (a visit, a vaccination) + * gets the new ids and adds them itself. Linking on the server there would + * be undone by the form's own save, which replaces its links. + * + * With either, a document already in HealthLog can be picked too: it is not + * stored twice, it is linked or handed to the form. + */ +import { + useInfiniteQuery, + useQuery, + useQueryClient, +} from "@tanstack/react-query"; +import { + Check, + CircleAlert, + CircleCheck, + Loader2, + Search, + SearchX, + X, +} from "lucide-react"; +import { useEffect, useMemo, useRef, useState } from "react"; + +import { Badge } from "@/components/ui/badge"; +import { Button } from "@/components/ui/button"; +import { EmptyState } from "@/components/ui/empty-state"; +import { DateField } from "@/components/ui/date-field"; +import { Input } from "@/components/ui/input"; +import { Label } from "@/components/ui/label"; +import { NativeSelect } from "@/components/ui/native-select"; +import { QueryErrorRow } from "@/components/ui/query-error-row"; +import { ResponsiveSheet } from "@/components/ui/responsive-sheet"; +import { Skeleton } from "@/components/ui/skeleton"; +import { TagChip } from "@/components/ui/tag-chip"; +import { apiGet, apiPost } from "@/lib/api/api-fetch"; +import { + DOCUMENT_PICKER_MAX_SELECTION, + slugForSystem, + type DocumentImportOutcome, + type DocumentPickerLinkKind, + type DocumentPickerSystem, + type DocumentSourceImportDto, + type DocumentSourceResultDto, + type DocumentSourceSearchDto, + type DocumentSourceTagDto, +} from "@/lib/documents/sources/types"; +import { useFormatters, useTranslations } from "@/lib/i18n/context"; +import { invalidateKeys, queryKeys } from "@/lib/query-keys"; +import { cn } from "@/lib/utils"; +import type { InboundDocumentKindValue } from "@/lib/validations/inbound-documents"; + +import { WrittenOutcomeLine } from "@/components/outcome/written-outcome-line"; +import type { WrittenOutcome } from "@/lib/outcome/written-outcome"; + +import { formatBytes } from "../vault-utils"; +import { + errorCodeOf, + isStoppingError, + sourceErrorMessage, +} from "./source-errors"; +import { SegmentedChoice } from "./segmented-choice"; +import { systemName } from "./use-document-sources"; + +export interface DocumentPickerLink { + kind: DocumentPickerLinkKind; + id: string; +} + +export interface DocumentSourcePickerProps { + open: boolean; + onOpenChange: (open: boolean) => void; + /** Connected systems, in the order the switch shows them. */ + systems: DocumentPickerSystem[]; + /** File the imported documents against this existing record. */ + link?: DocumentPickerLink; + /** Document type for everything imported (default OTHER). */ + kind?: InboundDocumentKindValue; + /** Hand the vault ids to a form that links them on its own save. */ + onImported?: (documentIds: string[]) => void; +} + +type RowRun = + | { state: "waiting" } + | { state: "working" } + | { state: DocumentImportOutcome; linked: boolean } + | { state: "failed"; message: string }; + +const SEARCH_DEBOUNCE_MS = 300; +const IMPORT_TIMEOUT_MS = 120_000; + +export function DocumentSourcePicker({ + open, + onOpenChange, + systems, + link, + kind, + onImported, +}: DocumentSourcePickerProps) { + const { t, locale } = useTranslations(); + const format = useFormatters(); + const queryClient = useQueryClient(); + + const [system, setSystem] = useState(null); + const active = + system && systems.includes(system) ? system : (systems[0] ?? null); + const slug = active ? slugForSystem(active) : null; + + const [searchDraft, setSearchDraft] = useState(""); + const [q, setQ] = useState(""); + const [tag, setTag] = useState(""); + const [from, setFrom] = useState(""); + const [to, setTo] = useState(""); + const [selected, setSelected] = useState>(new Map()); + const [runs, setRuns] = useState>(new Map()); + const [running, setRunning] = useState(false); + const [finished, setFinished] = useState(false); + const [stopMessage, setStopMessage] = useState(null); + const [atMax, setAtMax] = useState(false); + /** The latest per-row result, for screen readers (aria-live below). */ + const [announcement, setAnnouncement] = useState(""); + + // A new search or filter starts over: a selection made in one result list + // must not ride along unseen into another, and a finished run's result + // lines belong to the list they were made in. + const startOver = () => { + setSelected(new Map()); + setRuns(new Map()); + setFinished(false); + setStopMessage(null); + setAtMax(false); + }; + + const debounceRef = useRef | null>(null); + useEffect( + () => () => { + if (debounceRef.current) clearTimeout(debounceRef.current); + }, + [], + ); + const onSearchChange = (value: string) => { + setSearchDraft(value); + if (debounceRef.current) clearTimeout(debounceRef.current); + debounceRef.current = setTimeout(() => { + const next = value.trim(); + if (next !== q) { + startOver(); + setQ(next); + } + }, SEARCH_DEBOUNCE_MS); + }; + const filterSetter = + (set: (value: string) => void) => + (value: string): void => { + startOver(); + set(value); + }; + + const reset = () => { + setSearchDraft(""); + setQ(""); + setTag(""); + setFrom(""); + setTo(""); + startOver(); + }; + + const switchSystem = (next: DocumentPickerSystem) => { + if (next === active || running) return; + reset(); + setSystem(next); + }; + + const tags = useQuery({ + queryKey: queryKeys.documentSourceTags(slug ?? ""), + enabled: open && slug !== null, + queryFn: () => + apiGet<{ tags: DocumentSourceTagDto[] }>( + `/api/documents/sources/${slug}/tags`, + ), + staleTime: 5 * 60_000, + }); + + const facets = { + q, + ...(tag ? { tag } : {}), + ...(from ? { from } : {}), + ...(to ? { to } : {}), + }; + const rangeInvalid = Boolean(from && to && from > to); + const search = useInfiniteQuery({ + queryKey: queryKeys.documentSourceSearch(slug ?? "", facets), + enabled: open && slug !== null && !rangeInvalid, + initialPageParam: 1, + queryFn: ({ pageParam }) => { + const sp = new URLSearchParams({ page: String(pageParam) }); + if (q) sp.set("q", q); + if (tag) sp.set("tag", tag); + if (from) sp.set("from", from); + if (to) sp.set("to", to); + return apiGet( + `/api/documents/sources/${slug}/search?${sp.toString()}`, + ); + }, + getNextPageParam: (last) => (last.hasMore ? last.page + 1 : undefined), + staleTime: 30_000, + }); + + const results = useMemo(() => { + const seen = new Set(); + const out: DocumentSourceResultDto[] = []; + for (const page of search.data?.pages ?? []) { + for (const row of page.results) { + if (seen.has(row.sourceId)) continue; + seen.add(row.sourceId); + out.push(row); + } + } + return out; + }, [search.data]); + + // An already imported document is worth picking only when the pick does + // something with it: link it, or hand it to the form. + const canTakeImported = Boolean(link || onImported); + const selectable = (row: DocumentSourceResultDto) => + row.state === "new" || (row.state === "imported" && canTakeImported); + + const toggle = (row: DocumentSourceResultDto) => { + if (running || finished || !selectable(row)) return; + const next = new Map(selected); + if (next.has(row.sourceId)) { + next.delete(row.sourceId); + setAtMax(false); + } else if (next.size < DOCUMENT_PICKER_MAX_SELECTION) { + next.set(row.sourceId, row.title); + } else { + // Said rather than silently ignored: the tick that did not appear. + setAtMax(true); + return; + } + setSelected(next); + }; + + async function runImport() { + if (!slug || selected.size === 0) return; + setRunning(true); + setStopMessage(null); + const order = [...selected.keys()]; + setRuns(new Map(order.map((id) => [id, { state: "waiting" }]))); + const handed: string[] = []; + const mark = (id: string, run: RowRun) => { + setRuns((prev) => new Map(prev).set(id, run)); + const title = selected.get(id) ?? id; + const said = runLabel(t, run); + if (said) setAnnouncement(`${title}: ${said}`); + }; + + for (const sourceId of order) { + mark(sourceId, { state: "working" }); + try { + const result = await apiPost( + `/api/documents/sources/${slug}/import`, + { + sourceId, + ...(kind ? { kind } : {}), + ...(link ? { link } : {}), + }, + // The server may take up to a minute to fetch a large file from + // the source; the default 15 s client timeout would abandon it. + { signal: AbortSignal.timeout(IMPORT_TIMEOUT_MS) }, + ); + mark(sourceId, { state: result.outcome, linked: result.linked }); + if (result.outcome !== "deleted" && result.documentId) { + handed.push(result.documentId); + } + } catch (err) { + const code = errorCodeOf(err); + const message = sourceErrorMessage(t, code); + mark(sourceId, { state: "failed", message }); + if (isStoppingError(code)) { + setStopMessage(message); + break; + } + } + } + + setRunning(false); + setFinished(true); + setSelected(new Map()); + // The vault, its pickers and the record's own lists read documents + // through the `["documents"]` prefix; the picker's states through its own. + void invalidateKeys(queryClient, [ + queryKeys.documents(), + queryKeys.documentSources(), + ]); + if (handed.length > 0) onImported?.(handed); + } + + const counts = useMemo(() => { + let imported = 0; + let present = 0; + let failed = 0; + for (const run of runs.values()) { + if (run.state === "imported") imported += 1; + else if (run.state === "duplicate" || run.state === "deleted") + present += 1; + else if (run.state === "failed") failed += 1; + } + return { imported, present, failed }; + }, [runs]); + + const name = active ? systemName(active) : ""; + const hasFilters = Boolean(tag || from || to); + + const footer = ( +
+

+ {finished + ? null + : running + ? t("documents.sourcePicker.importing") + : t("documents.sourcePicker.selectedCount", { + count: selected.size, + max: DOCUMENT_PICKER_MAX_SELECTION, + })} +

+ {finished ? ( +
+ + +
+ ) : ( + + )} +
+ ); + + return ( + { + // A run in progress finishes before the sheet can close; its rows + // are the only place the outcome is shown. + if (!next && running) return; + if (!next) reset(); + onOpenChange(next); + }} + title={t("documents.sourcePicker.title", { name })} + description={t("documents.sourcePicker.description")} + contentWidth="lg" + footer={footer} + > +
+ {systems.length > 1 && active ? ( + + ) : null} + +
+ +
+ + onSearchChange(e.target.value)} + placeholder={t("documents.sourcePicker.searchPlaceholder")} + className="pl-9" + maxLength={200} + autoComplete="off" + data-slot="document-source-search" + /> +
+
+ +
+
+ + filterSetter(setTag)(e.target.value)} + disabled={tags.isPending && !tags.isError} + > + + {(tags.data?.tags ?? []).map((option) => ( + + ))} + +
+
+ + +
+
+ + +
+
+ {hasFilters ? ( + + ) : null} + + {finished ? ( +
+ +

+ {t("documents.sourcePicker.finishedHint")} +

+
+ ) : null} + + {atMax ? ( +

+ {t("documents.sourcePicker.maxReached", { + max: DOCUMENT_PICKER_MAX_SELECTION, + })} +

+ ) : null} + + {/* Each row's result as it lands, for a screen reader. */} +
+ {announcement} +
+ + {stopMessage ? ( +

+ {t("documents.sourcePicker.stopped", { reason: stopMessage })} +

+ ) : null} + + {rangeInvalid ? ( +

+ {t("documents.sourcePicker.rangeInvalid")} +

+ ) : search.isPending ? ( +
+ {Array.from({ length: 4 }, (_, i) => ( + + ))} +
+ ) : search.isError ? ( + void search.refetch()} + slot="document-source-error" + /> + ) : results.length === 0 ? ( + } + title={ + q || hasFilters + ? t("documents.sourcePicker.noMatch") + : t("documents.sourcePicker.empty", { name }) + } + description={ + q || hasFilters + ? t("documents.sourcePicker.noMatchHint") + : undefined + } + /> + ) : ( +
    + {results.map((row) => { + const on = selected.has(row.sourceId); + const run = runs.get(row.sourceId); + const allowed = selectable(row); + const meta = [ + row.date ? format.date(`${row.date}T12:00:00.000Z`) : null, + row.sizeBytes !== null + ? formatBytes(row.sizeBytes, locale) + : null, + ] + .filter(Boolean) + .join(" · "); + return ( +
  • + +
    • + ); + })} +
+ )} + + {search.hasNextPage && !search.isError ? ( + + ) : null} +
+ + ); +} + +/** How the run went, in the shared outcome vocabulary. */ +function runOutcome(counts: { + imported: number; + present: number; + failed: number; +}): WrittenOutcome { + const held = counts.imported + counts.present; + if (counts.failed === 0) return held > 0 ? "success" : "empty"; + return held > 0 ? "partial" : "failed"; +} + +/** A row's result in words, or null while it has none yet. */ +function runLabel( + t: (key: string, params?: Record) => string, + run: RowRun, +): string | null { + switch (run.state) { + case "waiting": + case "working": + return null; + case "failed": + return run.message; + case "deleted": + return t("documents.sourcePicker.result.deleted"); + case "imported": + return run.linked + ? t("documents.sourcePicker.result.importedLinked") + : t("documents.sourcePicker.result.imported"); + case "duplicate": + return run.linked + ? t("documents.sourcePicker.result.duplicateLinked") + : t("documents.sourcePicker.result.duplicate"); + } +} + +/** One row's progress or result, under its title. */ +function RunLine({ run }: { run: RowRun }) { + const { t } = useTranslations(); + switch (run.state) { + case "waiting": + return ( + + {t("documents.sourcePicker.result.waiting")} + + ); + case "working": + return ( + + + {t("documents.sourcePicker.result.working")} + + ); + case "failed": + return ( + + + {run.message} + + ); + case "deleted": + return ( + + {t("documents.sourcePicker.result.deleted")} + + ); + case "imported": + case "duplicate": + return ( + + + {run.state === "imported" + ? run.linked + ? t("documents.sourcePicker.result.importedLinked") + : t("documents.sourcePicker.result.imported") + : run.linked + ? t("documents.sourcePicker.result.duplicateLinked") + : t("documents.sourcePicker.result.duplicate")} + + ); + } +} diff --git a/src/components/documents/sources/import-from-source-button.tsx b/src/components/documents/sources/import-from-source-button.tsx new file mode 100644 index 000000000..09fd872df --- /dev/null +++ b/src/components/documents/sources/import-from-source-button.tsx @@ -0,0 +1,73 @@ +"use client"; + +/** + * "Import from Paperless-ngx / Papra": the button that opens the document + * picker (#1038), for every place a document can be added or linked. + * + * Renders nothing unless the person, in their own record, has at least one + * usable connection: no connection, a delegate or guardian, the module off, or + * the operator's list unset all leave no trace, because a button that can only + * lead to a refusal advertises a feature that is not there. Connecting happens + * in Settings, not here. + */ +import { Import } from "lucide-react"; +import { useState } from "react"; + +import { Button } from "@/components/ui/button"; +import { useTranslations } from "@/lib/i18n/context"; +import type { InboundDocumentKindValue } from "@/lib/validations/inbound-documents"; + +import { + DocumentSourcePicker, + type DocumentPickerLink, +} from "./document-source-picker"; +import { systemName, useDocumentSourcesStatus } from "./use-document-sources"; + +export function ImportFromSourceButton({ + link, + kind, + onImported, + className, +}: { + link?: DocumentPickerLink; + kind?: InboundDocumentKindValue; + onImported?: (documentIds: string[]) => void; + className?: string; +}) { + const { t } = useTranslations(); + const { connected } = useDocumentSourcesStatus(); + const [open, setOpen] = useState(false); + + if (connected.length === 0) return null; + + const label = + connected.length === 1 + ? t("documents.sourcePicker.openNamed", { + name: systemName(connected[0]), + }) + : t("documents.sourcePicker.open"); + + return ( + <> + + + + ); +} diff --git a/src/components/documents/sources/merge-ids.ts b/src/components/documents/sources/merge-ids.ts new file mode 100644 index 000000000..ab8579349 --- /dev/null +++ b/src/components/documents/sources/merge-ids.ts @@ -0,0 +1,6 @@ +/** Add imported ids to a form's selection, in order, never twice. */ +export function mergeIds(selected: string[], added: string[]): string[] { + const merged = [...selected]; + for (const id of added) if (!merged.includes(id)) merged.push(id); + return merged; +} diff --git a/src/components/documents/sources/segmented-choice.tsx b/src/components/documents/sources/segmented-choice.tsx new file mode 100644 index 000000000..a140cfa5e --- /dev/null +++ b/src/components/documents/sources/segmented-choice.tsx @@ -0,0 +1,110 @@ +"use client"; + +/** + * A small segmented radio group (Paperless-ngx | Papra) with the keyboard + * contract of a radio group: one tab stop (the checked option), arrow keys + * move and select, Home and End jump to the ends. Shared by the document + * picker's source switch and the settings card's system switch. + */ +import { useRef, type KeyboardEvent, type ReactNode } from "react"; + +import { cn } from "@/lib/utils"; + +/** + * The option an arrow, Home or End key moves to from `index`, wrapping at the + * ends; null for any other key. + */ +export function nextChoiceIndex( + key: string, + index: number, + count: number, +): number | null { + if (count <= 0) return null; + switch (key) { + case "ArrowRight": + case "ArrowDown": + return (index + 1) % count; + case "ArrowLeft": + case "ArrowUp": + return (index - 1 + count) % count; + case "Home": + return 0; + case "End": + return count - 1; + default: + return null; + } +} + +export function SegmentedChoice({ + options, + value, + onChange, + label, + disabled = false, + renderOption, + slot, + className, +}: { + options: readonly T[]; + value: T; + onChange: (value: T) => void; + /** Accessible name of the group. */ + label: string; + disabled?: boolean; + renderOption: (option: T) => ReactNode; + slot?: string; + className?: string; +}) { + const refs = useRef>([]); + + const onKeyDown = (event: KeyboardEvent, i: number) => { + const next = nextChoiceIndex(event.key, i, options.length); + if (next === null) return; + event.preventDefault(); + if (disabled) return; + onChange(options[next]); + refs.current[next]?.focus(); + }; + + return ( +
+ {options.map((option, i) => { + const checked = option === value; + return ( + + ); + })} +
+ ); +} diff --git a/src/components/documents/sources/source-errors.ts b/src/components/documents/sources/source-errors.ts new file mode 100644 index 000000000..9ec63e293 --- /dev/null +++ b/src/components/documents/sources/source-errors.ts @@ -0,0 +1,84 @@ +/** + * The document picker's error codes (#1038), as sentences. One place for the + * sheet and the settings card, so the same refusal reads the same on both. + * Every branch names its key literally so the i18n coverage guard can see it. + */ +import { ApiError } from "@/lib/api/api-fetch"; + +type T = (key: string, params?: Record) => string; + +/** The `meta.errorCode` a failed request carried, if any. */ +export function errorCodeOf(err: unknown): string | undefined { + if (err instanceof ApiError) { + const code = err.meta?.errorCode; + if (typeof code === "string") return code; + } + return undefined; +} + +/** Is this a refusal that will not change on the next document? */ +export function isStoppingError(code: string | undefined): boolean { + return ( + code === "documents.inbound.rateLimited" || + code === "documents.inbound.quotaExceeded" || + code === "documents.sources.authRefused" || + code === "documents.sources.permissionMissing" || + code === "documents.sources.unreachable" || + code === "documents.sources.originNotAllowed" || + code === "documents.sources.notConnected" || + code === "documents.sources.unavailable" || + code === "documents.sources.browserOnly" || + code === "documents.sources.linkTargetNotFound" + ); +} + +export function sourceErrorMessage(t: T, code: string | undefined): string { + switch (code) { + case "documents.sources.unavailable": + return t("documents.sourcePicker.errors.unavailable"); + case "documents.sources.notConnected": + return t("documents.sourcePicker.errors.notConnected"); + case "documents.sources.originNotAllowed": + return t("documents.sourcePicker.errors.originNotAllowed"); + case "documents.sources.unreachable": + return t("documents.sourcePicker.errors.unreachable"); + case "documents.sources.redirected": + return t("documents.sourcePicker.errors.redirected"); + case "documents.sources.authRefused": + return t("documents.sourcePicker.errors.authRefused"); + case "documents.sources.permissionMissing": + return t("documents.sourcePicker.errors.permissionMissing"); + case "documents.sources.versionTooOld": + return t("documents.sourcePicker.errors.versionTooOld"); + case "documents.sources.badResponse": + return t("documents.sourcePicker.errors.badResponse"); + case "documents.sources.notFound": + return t("documents.sourcePicker.errors.notFound"); + case "documents.sources.linkTargetNotFound": + return t("documents.sourcePicker.errors.linkTargetNotFound"); + case "documents.sources.rateLimited": + return t("documents.sourcePicker.errors.rateLimited"); + case "documents.sources.browserOnly": + return t("documents.sourcePicker.errors.browserOnly"); + case "documents.sources.invalidAddress": + return t("documents.sourcePicker.errors.invalidAddress"); + case "documents.sources.organizationRequired": + return t("documents.sourcePicker.errors.organizationRequired"); + case "documents.sources.tokenRequired": + return t("documents.sourcePicker.errors.tokenRequired"); + case "documents.inbound.fileTooLarge": + return t("documents.sourcePicker.errors.fileTooLarge"); + case "documents.inbound.quotaExceeded": + return t("documents.sourcePicker.errors.quotaExceeded"); + case "documents.inbound.fileType": + return t("documents.sourcePicker.errors.fileType"); + case "documents.inbound.rateLimited": + return t("documents.sourcePicker.errors.uploadLimit"); + case "documents.inbound.uploadBusy": + return t("documents.sourcePicker.errors.uploadBusy"); + case "module.disabled": + return t("documents.sourcePicker.errors.moduleDisabled"); + default: + return t("documents.sourcePicker.errors.generic"); + } +} diff --git a/src/components/documents/sources/use-document-sources.ts b/src/components/documents/sources/use-document-sources.ts new file mode 100644 index 000000000..e19329ca2 --- /dev/null +++ b/src/components/documents/sources/use-document-sources.ts @@ -0,0 +1,58 @@ +"use client"; + +/** + * The document picker's availability for the person on this screen (#1038). + * + * Reads `GET /api/documents/sources` only in the person's own record: a + * delegate or guardian never asks (the route would refuse them anyway), so + * the import buttons and the settings card render nothing for them. + * `connected` is the list of systems with a usable connection (the documents + * module on, the operator's list set, the origin still on it), in a stable + * order; the import buttons read only that. + */ +import { useQuery } from "@tanstack/react-query"; + +import { useAuth } from "@/hooks/use-auth"; +import { useRecordCapabilities } from "@/hooks/use-record-capabilities"; +import { apiGet } from "@/lib/api/api-fetch"; +import type { + DocumentPickerSystem, + DocumentSourcesStatusDto, +} from "@/lib/documents/sources/types"; +import { queryKeys } from "@/lib/query-keys"; + +export function useDocumentSourcesStatus() { + const { user } = useAuth(); + const { canManageDomain, inSharedRecord } = useRecordCapabilities(); + // The status read touches only the person's own rows and never the + // network, so it runs in one's own record whatever the module says: a + // connection stored before Documents was switched off, or before the + // operator removed the list, must stay visible so it can be deleted. + const ownRecord = + user !== null && + user !== undefined && + !inSharedRecord && + canManageDomain("documents"); + const eligible = ownRecord && user?.modules?.inboundDocuments === true; + + const query = useQuery({ + queryKey: queryKeys.documentSourcesStatus(), + enabled: ownRecord, + queryFn: () => apiGet("/api/documents/sources"), + staleTime: 60_000, + }); + + const connected: DocumentPickerSystem[] = + eligible && query.data?.available + ? query.data.connections + .filter((c) => c.originAllowed) + .map((c) => c.system) + : []; + + return { ownRecord, eligible, query, connected }; +} + +/** Display name of a system; product names, not translated. */ +export function systemName(system: DocumentPickerSystem): string { + return system === "PAPERLESS" ? "Paperless-ngx" : "Papra"; +} diff --git a/src/components/encounters/encounter-link-pickers.tsx b/src/components/encounters/encounter-link-pickers.tsx index 1dcf1e536..1e0eba0fe 100644 --- a/src/components/encounters/encounter-link-pickers.tsx +++ b/src/components/encounters/encounter-link-pickers.tsx @@ -32,6 +32,8 @@ import { type EntityLinkOption, } from "@/components/links/entity-link-picker"; import { useVaultDocumentOptions } from "@/components/links/vault-document-options"; +import { ImportFromSourceButton } from "@/components/documents/sources/import-from-source-button"; +import { mergeIds } from "@/components/documents/sources/merge-ids"; /** Fetch cap for the picker — the grouped sheet needs enough rows to group. */ export const PICKER_FETCH_LIMIT = 200; @@ -152,6 +154,17 @@ export function EncounterLinkPickers({ options={documents.options} searchPlaceholder={t("links.picker.searchPlaceholder")} emptyLabel={t("encounters.form.linkNothingToOffer")} + extraAction={ + // The form owns its selection and replaces the visit's links on + // save, so imported documents join the selection rather than + // being linked on the server. + + onChange({ documentIds: mergeIds(documentIds, ids) }) + } + /> + } /> ) : null} diff --git a/src/components/links/entity-link-picker.tsx b/src/components/links/entity-link-picker.tsx index 3b3f4a467..8d70d10b0 100644 --- a/src/components/links/entity-link-picker.tsx +++ b/src/components/links/entity-link-picker.tsx @@ -31,7 +31,7 @@ * carries the option's date, so two doses of one vaccine read as two * different records. */ -import { useMemo, useState } from "react"; +import { useMemo, useState, type ReactNode } from "react"; import Link from "next/link"; import { Check, Plus, Search, X } from "lucide-react"; import type { LucideIcon } from "lucide-react"; @@ -164,6 +164,7 @@ export function EntityLinkPicker({ error = false, errorLabel, onRetry, + extraAction, }: { icon: LucideIcon; title: string; @@ -183,6 +184,13 @@ export function EntityLinkPicker({ error?: boolean; errorLabel?: string; onRetry?: () => void; + /** + * A second way to add, beside the add button: the documents pickers put + * "Import from Paperless-ngx / Papra" here. Shown even when there is + * nothing to link yet, because an empty vault is exactly when importing is + * the way forward. The element renders nothing when it has nothing to offer. + */ + extraAction?: ReactNode; }) { const { t } = useTranslations(); const [open, setOpen] = useState(false); @@ -238,7 +246,10 @@ export function EntityLinkPicker({ slot={`${slot}-error`} /> ) : options.length === 0 ? ( -

{emptyLabel}

+
+

{emptyLabel}

+ {extraAction} +
) : (
{selectedChips.length > 0 ? ( @@ -312,17 +323,20 @@ export function EntityLinkPicker({

) : null} - +
+ + {extraAction} +
)} diff --git a/src/components/settings/integrations-section.tsx b/src/components/settings/integrations-section.tsx index 37ceed391..4b6ce1506 100644 --- a/src/components/settings/integrations-section.tsx +++ b/src/components/settings/integrations-section.tsx @@ -37,6 +37,7 @@ import { ConnectionsPanel } from "@/components/settings/integrations/connections import { SectionHeading } from "@/components/ui/section-heading"; import { NotificationChannelsPanel } from "@/components/settings/integrations/notification-channels-panel"; import { NutrientIntakeCard } from "@/components/settings/nutrient-intake-card"; +import { DocumentSourcesCard } from "@/components/settings/integrations/document-sources-card"; import { Button } from "@/components/ui/button"; export { @@ -77,6 +78,11 @@ export function IntegrationsSection({ under the same Connections group rather than a new labelled section that could paint an orphaned heading over nothing. */} + + {/* v1.39.3 (#1038) — Paperless-ngx / Papra for the document picker. + Renders nothing unless the operator listed an origin in + DOCUMENT_SOURCE_ORIGINS and the person is in their own record. */} + {/* Delivery channels ("where it's delivered"). The existing channels diff --git a/src/components/settings/integrations/document-sources-card.tsx b/src/components/settings/integrations/document-sources-card.tsx new file mode 100644 index 000000000..8a0cf216e --- /dev/null +++ b/src/components/settings/integrations/document-sources-card.tsx @@ -0,0 +1,453 @@ +"use client"; + +/** + * Settings → Integrations → Document archives (#1038): connect Paperless-ngx or + * Papra so the document picker can search it. + * + * Renders nothing unless the operator listed at least one origin in + * `DOCUMENT_SOURCE_ORIGINS`, the documents module is on, and the person is in + * their own record: a delegate never manages the owner's connection, and a + * card that can only be refused advertises a feature that is not there. + * + * One form per system behind a switch. Save tests the connection before it is + * stored and says why when it fails; the token field never shows the stored + * token (the server never returns it), only that one is saved, and leaving it + * empty keeps it. Results sit above the action row; nothing follows it + * (UI standards §12). + */ +import { useMutation, useQueryClient } from "@tanstack/react-query"; +import { Archive, Check, Loader2, Unlink } from "lucide-react"; +import { useState } from "react"; + +import { + AlertDialog, + AlertDialogAction, + AlertDialogCancel, + AlertDialogContent, + AlertDialogDescription, + AlertDialogFooter, + AlertDialogHeader, + AlertDialogTitle, + AlertDialogTrigger, +} from "@/components/ui/alert-dialog"; +import { Badge } from "@/components/ui/badge"; +import { Button } from "@/components/ui/button"; +import { Input } from "@/components/ui/input"; +import { Label } from "@/components/ui/label"; +import { PasswordInput } from "@/components/ui/password-input"; +import { SettingsCardActions } from "@/components/settings/_card-actions"; +import { SettingsCardHeader } from "@/components/settings/_card-header"; +import { SettingsCard } from "@/components/settings/settings-card"; +import { WrittenOutcomeLine } from "@/components/outcome/written-outcome-line"; +import { SegmentedChoice } from "@/components/documents/sources/segmented-choice"; +import { + errorCodeOf, + sourceErrorMessage, +} from "@/components/documents/sources/source-errors"; +import { + systemName, + useDocumentSourcesStatus, +} from "@/components/documents/sources/use-document-sources"; +import { apiDelete, apiPost, apiPut } from "@/lib/api/api-fetch"; +import { + DOCUMENT_PICKER_SYSTEMS, + slugForSystem, + type DocumentPickerSystem, + type DocumentSourceConnectionDto, +} from "@/lib/documents/sources/types"; +import { useFormatters, useTranslations } from "@/lib/i18n/context"; +import { invalidateKeys, queryKeys } from "@/lib/query-keys"; + +type Outcome = { tone: "success" | "error"; message: string } | null; + +export function DocumentSourcesCard() { + const { t } = useTranslations(); + const { ownRecord, eligible, query } = useDocumentSourcesStatus(); + const [picked, setPicked] = useState("PAPERLESS"); + + const data = query.data; + const connections = data?.connections ?? []; + // Editing needs the operator's list and the documents module. Without + // either, a connection that is still stored stays on the card so its token + // can be deleted; with none stored there is nothing to show. + const editable = eligible && data?.available === true; + if (!ownRecord || !data || (!editable && connections.length === 0)) { + return null; + } + + const options: readonly DocumentPickerSystem[] = editable + ? DOCUMENT_PICKER_SYSTEMS + : connections.map((c) => c.system); + const system = options.includes(picked) ? picked : options[0]; + + return ( + + 0 ? ( + + {t("settings.documentSources.statusConnected")} + + ) : null + } + /> + +

+ {editable + ? t("settings.documentSources.explainer") + : eligible + ? t("settings.documentSources.unavailableExplainer") + : t("settings.documentSources.moduleOffExplainer")} +

+ + {options.length > 1 ? ( + ( + <> + {systemName(option)} + {connections.some((c) => c.system === option) ? ( + + ) : null} + + )} + /> + ) : null} + + c.system === system) ?? null} + allowedOrigins={data.allowedOrigins} + editable={editable} + /> + + ); +} + +/** The origin of an address, or null while it is not one yet. */ +function originOf(value: string): string | null { + try { + return new URL(value.trim()).origin; + } catch { + return null; + } +} + +function SourceForm({ + system, + connection, + allowedOrigins, + editable, +}: { + system: DocumentPickerSystem; + connection: DocumentSourceConnectionDto | null; + allowedOrigins: string[]; + /** False when only removing the stored connection is possible. */ + editable: boolean; +}) { + const { t } = useTranslations(); + const format = useFormatters(); + const queryClient = useQueryClient(); + const slug = slugForSystem(system); + const isPapra = system === "PAPRA"; + + const [baseUrl, setBaseUrl] = useState(connection?.baseUrl ?? ""); + const [organizationId, setOrganizationId] = useState( + connection?.organizationId ?? "", + ); + const [token, setToken] = useState(""); + const [outcome, setOutcome] = useState(null); + + const dirty = + baseUrl.trim() !== (connection?.baseUrl ?? "") || + (isPapra && organizationId.trim() !== (connection?.organizationId ?? "")) || + token.trim() !== ""; + + const refresh = () => + invalidateKeys(queryClient, [queryKeys.documentSources()]); + + const save = useMutation({ + mutationFn: () => + apiPut(`/api/documents/sources/${slug}`, { + baseUrl: baseUrl.trim(), + ...(isPapra ? { organizationId: organizationId.trim() } : {}), + ...(token.trim() ? { token: token.trim() } : {}), + }), + onMutate: () => setOutcome(null), + onSuccess: () => { + setToken(""); + setOutcome({ + tone: "success", + message: t("settings.documentSources.saved"), + }); + void refresh(); + }, + onError: (err) => + setOutcome({ + tone: "error", + message: sourceErrorMessage(t, errorCodeOf(err)), + }), + }); + + const test = useMutation({ + mutationFn: () => + apiPost<{ ok: true; latencyMs: number }>( + `/api/documents/sources/${slug}/test`, + ), + onMutate: () => setOutcome(null), + onSuccess: (data) => { + setOutcome({ + tone: "success", + message: t("settings.documentSources.testOk", { + latency: data.latencyMs, + }), + }); + void refresh(); + }, + onError: (err) => + setOutcome({ + tone: "error", + message: sourceErrorMessage(t, errorCodeOf(err)), + }), + }); + + const disconnect = useMutation({ + mutationFn: () => apiDelete(`/api/documents/sources/${slug}`), + onSuccess: () => { + setBaseUrl(""); + setOrganizationId(""); + setToken(""); + setOutcome(null); + void refresh(); + }, + onError: (err) => + setOutcome({ + tone: "error", + message: sourceErrorMessage(t, errorCodeOf(err)), + }), + }); + + const busy = save.isPending || test.isPending || disconnect.isPending; + // The stored token is reused only on the address's own origin; the server + // refuses to send it anywhere else, so the form asks for it again. + const keepsToken = + connection !== null && + originOf(baseUrl) !== null && + originOf(baseUrl) === originOf(connection.baseUrl); + const canSave = + editable && + baseUrl.trim() !== "" && + (!isPapra || organizationId.trim() !== "") && + (keepsToken || token.trim() !== "") && + dirty; + + return ( +
{ + e.preventDefault(); + if (canSave && !busy) save.mutate(); + }} + data-slot="document-sources-form" + > +

+ {connection + ? connection.lastVerifiedAt + ? t("settings.documentSources.connectedSince", { + date: format.date(connection.lastVerifiedAt), + }) + : t("settings.documentSources.statusConnected") + : t("settings.documentSources.statusNotConnected")} +

+ + {editable && connection && !connection.originAllowed ? ( +

+ {t("settings.documentSources.originRemoved")} +

+ ) : null} + + {editable ? ( + <> +
+ + setBaseUrl(e.target.value)} + placeholder={allowedOrigins[0] ?? "https://"} + maxLength={2048} + autoComplete="off" + inputMode="url" + spellCheck={false} + autoCapitalize="none" + /> +

+ {t("settings.documentSources.allowedOrigins", { + origins: allowedOrigins.join(", "), + })} +

+
+ + {isPapra ? ( +
+ + setOrganizationId(e.target.value)} + maxLength={128} + autoComplete="off" + spellCheck={false} + autoCapitalize="none" + /> +

+ {t("settings.documentSources.organizationHelp")} +

+
+ ) : null} + +
+ + setToken(e.target.value)} + placeholder={ + keepsToken + ? t("settings.documentSources.tokenSaved") + : undefined + } + maxLength={512} + autoComplete="off" + spellCheck={false} + autoCapitalize="none" + /> +

+ {isPapra + ? t("settings.documentSources.tokenHelpPapra") + : t("settings.documentSources.tokenHelpPaperless")} +

+ {connection && !keepsToken && baseUrl.trim() !== "" ? ( +

+ {t("settings.documentSources.tokenNewAddress")} +

+ ) : null} +
+ + ) : connection ? ( +

+ {connection.baseUrl} +

+ ) : null} + + {outcome ? ( + + ) : null} + + + {connection ? ( + + + + + + + + {t("settings.documentSources.disconnectTitle", { + name: systemName(system), + })} + + + {t("settings.documentSources.disconnectDescription")} + + + + {t("common.cancel")} + disconnect.mutate()}> + {t("settings.documentSources.disconnect")} + + + + + ) : null} + {connection && editable ? ( + + ) : null} + {editable ? ( + + ) : null} + + + ); +} diff --git a/src/components/vaccinations/vaccination-document-picker.tsx b/src/components/vaccinations/vaccination-document-picker.tsx index d1778773b..6fb195261 100644 --- a/src/components/vaccinations/vaccination-document-picker.tsx +++ b/src/components/vaccinations/vaccination-document-picker.tsx @@ -28,6 +28,8 @@ import { FolderOpen } from "lucide-react"; import { useTranslations } from "@/lib/i18n/context"; import { EntityLinkPicker } from "@/components/links/entity-link-picker"; import { useVaultDocumentOptions } from "@/components/links/vault-document-options"; +import { ImportFromSourceButton } from "@/components/documents/sources/import-from-source-button"; +import { mergeIds } from "@/components/documents/sources/merge-ids"; export function VaccinationDocumentPicker({ enabled, @@ -73,6 +75,14 @@ export function VaccinationDocumentPicker({ options={vault.options} searchPlaceholder={t("links.picker.searchPlaceholder")} emptyLabel={t("vaccinations.form.linkNothingToOffer")} + extraAction={ + // Joins the form's selection; the dose's save writes the links. + onChange(mergeIds(documentIds, ids))} + /> + } /> ); diff --git a/src/lib/crypto/encrypted-columns.ts b/src/lib/crypto/encrypted-columns.ts index 684228dba..6759e5d7e 100644 --- a/src/lib/crypto/encrypted-columns.ts +++ b/src/lib/crypto/encrypted-columns.ts @@ -161,6 +161,14 @@ export const ENCRYPTED_COLUMNS: readonly EncryptedColumn[] = [ { model: "GoogleHealthConnection", field: "accessToken", kind: "string" }, { model: "GoogleHealthConnection", field: "refreshToken", kind: "string" }, + // ───── Document picker (v1.39.3, #1038) ───── + // A person's API token for their Paperless-ngx or Papra instance. + { + model: "DocumentSourceConnection", + field: "tokenEncrypted", + kind: "string", + }, + // ───── AppSettings — operator credentials ───── { model: "AppSettings", field: "adminAiKeyEncrypted", kind: "string" }, { diff --git a/src/lib/data-wipe/wipe-plan.ts b/src/lib/data-wipe/wipe-plan.ts index 3bb1bcf60..1e4f2ff9e 100644 --- a/src/lib/data-wipe/wipe-plan.ts +++ b/src/lib/data-wipe/wipe-plan.ts @@ -155,6 +155,10 @@ export const WIPE_MODELS = [ // Further import keys that resolved to a document by content; a child of // the document, removed before it so the counts are truthful. "DocumentSourceAlias", + // The person's own connections to Paperless-ngx / Papra (#1038), with their + // encrypted API tokens. A wipe that left them would leave a working + // credential to another system behind an account that erased everything. + "DocumentSourceConnection", "ExtractedFact", "ImportJob", diff --git a/src/lib/documents/ingest.ts b/src/lib/documents/ingest.ts new file mode 100644 index 000000000..0932ba813 --- /dev/null +++ b/src/lib/documents/ingest.ts @@ -0,0 +1,435 @@ +/** + * One way into the vault for a file that has already been read into memory. + * + * The upload route (`POST /api/documents/inbound`, cookie, `["*"]` or a + * `documents:write` token) and the document picker's import route + * (`POST /api/documents/sources/{system}/import`, which fetches a picked + * document from Paperless-ngx or Papra) both store through here, so the rules a + * stored document is held to cannot differ by the way it arrived: + * + * - a source key answers before the bytes are looked at: a key already held + * (live, tombstoned, remembered as an alias or kept in the purge ledger) is + * a duplicate or a "deleted", never a second copy; + * - the per-file cap, the empty-file refusal and magic-byte classification + * (the wire Content-Type is never trusted); + * - pre-links to the caller's own live episodes and visits, refused when an + * id names nothing the caller owns; + * - sha256 duplicate detection against live rows, remembering the source key + * for the existing document so a later delete keeps that key deleted too; + * for an import, also against deleted (not yet purged) rows, which answer + * "deleted" rather than storing the file again; + * - the quota gate and the insert in one transaction under a per-user + * advisory lock, and the race on either partial unique index resolved the + * way the fast paths above would have answered; + * - the audit row, the wide-event annotation, and the background index, + * thumbnail and summary jobs (the summary only when the `documentAi` + * capability is open and the upload did not ask to defer AI reading). + * + * The callers own authentication, rate buckets, request parsing and the shape + * of their responses. This module returns what happened; it builds no HTTP. + */ +import type { Buffer } from "node:buffer"; +import { createHash } from "node:crypto"; + +import type { AiCapabilityState } from "@/lib/ai/capabilities/types"; +import { auditLog } from "@/lib/auth/audit"; +import { prisma } from "@/lib/db"; +import { + narrowOwnedEncounterIds, + narrowOwnedEpisodeIds, +} from "@/lib/documents/links"; +import { + findSourceKey, + rememberSourceAlias, + type SourceKeyMatch, +} from "@/lib/documents/source-key"; +import { + encryptDocumentContent, + type SerialisableDocument, +} from "@/lib/documents/store"; +import { detectDocumentType } from "@/lib/documents/upload-policy"; +import { enqueueDocumentIndex } from "@/lib/jobs/document-index"; +import { enqueueDocumentSummary } from "@/lib/jobs/document-summary"; +import { enqueueDocumentThumbnail } from "@/lib/jobs/document-thumbnail"; +import { linkTargets } from "@/lib/links"; +import { annotate } from "@/lib/logging/context"; +import { isP2002 } from "@/lib/prisma-errors"; +import type { + DocumentSourceSystemValue, + InboundDocumentKindValue, +} from "@/lib/validations/inbound-documents"; + +/** + * The person's own uploads per hour, from the web or the phone, and the + * picker's imports with them. A store-only upload touches no provider — the + * only abuse vector is disk, so a generous ceiling is enough. A narrow + * `documents:write` token draws on its own bucket instead + * (`resolveDocumentUploadLimitPerHour`). + */ +export const UPLOAD_LIMIT_PER_HOUR = 60; +export const UPLOAD_WINDOW_MS = 60 * 60 * 1000; + +/** The bucket a person's own uploads and picker imports share. */ +export function personalUploadBucket(userId: string): string { + return `documents-upload:${userId}`; +} + +/** Parse a YYYY-MM-DD value into a UTC midnight Date. */ +export function isoDateToUtc(value: string): Date { + return new Date(`${value}T00:00:00.000Z`); +} + +export interface IngestInput { + userId: string; + /** A narrow `documents:write` token rather than a session or `["*"]`. */ + scoped: boolean; + /** For the audit row. */ + ipAddress: string | null; + bytes: Buffer; + filename: string | null; + title: string | null; + kind: InboundDocumentKindValue | null; + /** YYYY-MM-DD; the upload day when absent. */ + documentDate: string | null; + episodeIds: string[]; + encounterIds: string[]; + sourceSystem: DocumentSourceSystemValue | null; + sourceId: string | null; + /** + * The source instance (origin) of the key, or null for a key sent without + * one; see `src/lib/documents/source-key.ts` for how null matches. + */ + sourceInstance: string | null; + /** Hold back automatic AI reading (`aiRead=defer`). */ + aiDeferred: boolean; + /** + * The caller already asked about this source key (the upload's query-string + * form answers before the bucket is charged). Skips the second lookup. + */ + sourceKeyChecked: boolean; + /** The per-file cap and the quota, resolved by the caller. */ + limits: { maxFileBytes: number; quotaBytes: number }; + /** + * The `documentAi` capability, asked by the caller's route only once a new + * document was stored (a duplicate never asks). Passed in rather than asked + * here so each route that can queue AI work names the capability in its own + * source, where the AI route inventory reads it. + */ + documentAi: () => Promise; +} + +export type IngestResult = + | { kind: "sourceKey"; match: SourceKeyMatch } + | { kind: "tooLarge"; maxFileBytes: number } + | { kind: "empty" } + | { kind: "unsupportedType" } + | { kind: "episodeNotFound" } + | { kind: "encounterNotFound" } + | { kind: "aliasLimit" } + | { kind: "duplicate"; document: SerialisableDocument } + | { kind: "quotaExceeded"; usedBytes: number; quotaBytes: number } + | { + kind: "stored"; + document: SerialisableDocument; + servingClass: string; + linkedEpisodes: number; + linkedVisits: number; + }; + +/** Internal signal: the quota gate inside the insert transaction tripped. */ +class QuotaExceededError extends Error { + constructor(public readonly usedBytes: number) { + super("Document quota exceeded"); + this.name = "QuotaExceededError"; + } +} + +/** A live row with the same bytes, remembering the source key for it. */ +async function answerDuplicate( + input: IngestInput, + existing: SerialisableDocument, +): Promise { + // An import sending bytes that are already stored under another key (or + // none) is answered with that document — and the key is remembered for it, + // so once the person deletes the document this key stays deleted too. + if (input.sourceSystem && input.sourceId) { + const remembered = await rememberSourceAlias( + input.userId, + existing.id, + input.sourceSystem, + input.sourceId, + input.sourceInstance, + ); + if (remembered === "limit") return { kind: "aliasLimit" }; + } + return { kind: "duplicate", document: existing }; +} + +export async function ingestDocument( + input: IngestInput, +): Promise { + const { userId, sourceSystem, sourceId, limits, bytes } = input; + const sourceInstance = sourceId ? input.sourceInstance : null; + + // A source key answers before the bytes are looked at: an import re-sending + // what it sent before is a duplicate, and one re-sending a document the + // person deleted is refused a second copy — live and tombstoned rows both + // count (the unique index has no `deleted_at` predicate), and past the purge + // the ledger remembers. + if (sourceSystem && sourceId && !input.sourceKeyChecked) { + const match = await findSourceKey( + userId, + sourceSystem, + sourceId, + sourceInstance, + ); + if (match) return { kind: "sourceKey", match }; + } + + if (bytes.byteLength > limits.maxFileBytes) { + return { kind: "tooLarge", maxFileBytes: limits.maxFileBytes }; + } + if (bytes.byteLength === 0) return { kind: "empty" }; + + // Magic-byte classification; the wire Content-Type is never trusted. + const detected = detectDocumentType(bytes, input.filename); + if (!detected) return { kind: "unsupportedType" }; + + // Pre-linking: every episode id must be a LIVE episode of the caller. + const episodeIds = await narrowOwnedEpisodeIds(userId, input.episodeIds); + if (episodeIds === null) return { kind: "episodeNotFound" }; + + // Same for the visit ids the review step offered. A refusal rather than a + // silent drop: the person saw a visit named and would otherwise watch the + // upload succeed without the link they asked for. + const encounterIds = await narrowOwnedEncounterIds( + userId, + input.encounterIds, + ); + if (encounterIds === null) return { kind: "encounterNotFound" }; + + // sha256 of the PLAINTEXT for same-user duplicate detection. + const contentSha256 = createHash("sha256").update(bytes).digest("hex"); + + // Fast-path dedupe check; the partial unique index closes the race below. + const existing = await prisma.inboundDocument.findFirst({ + where: { userId, contentSha256, deletedAt: null }, + omit: { contentEncrypted: true }, + }); + if (existing) return answerDuplicate(input, existing); + + // An import (a source key is present) of bytes the person deleted here, + // arriving under a key HealthLog has not seen: another system, another + // instance, or a re-scan with a new id. The deletion was a decision about + // the document, not about one key, so it is answered "deleted", and the + // new key is remembered on the tombstone so the purge carries it into the + // ledger. After the 30-day purge the bytes and their hash are gone and only + // the keys remain; no content hash of a deleted document is kept. A person + // uploading the file by hand (no key) is making a new copy on purpose. + if (sourceSystem && sourceId) { + const tombstone = await prisma.inboundDocument.findFirst({ + where: { userId, contentSha256, deletedAt: { not: null } }, + select: { id: true }, + orderBy: { deletedAt: "desc" }, + }); + if (tombstone) { + const remembered = await rememberSourceAlias( + userId, + tombstone.id, + sourceSystem, + sourceId, + sourceInstance, + ); + if (remembered === "limit") return { kind: "aliasLimit" }; + return { + kind: "sourceKey", + match: { state: "deleted", id: tombstone.id }, + }; + } + } + + // Quota gate + insert + pre-links in ONE transaction. Usage counts every + // non-purged row — tombstones still hold TOAST bytes, so "deleted" bytes + // are never invisible weight (undo-delete never changes usage). + let document: SerialisableDocument; + try { + document = await prisma.$transaction(async (tx) => { + // Serialise the quota gate per user: without this, N concurrent + // uploads all read the same SUM before any of them commits and the + // quota can be overshot by up to N × cap in one burst. The advisory + // lock is transaction-scoped (released on commit/rollback) and keyed + // on the user id, so uploads by different users never queue on each + // other. + // (`pg_advisory_xact_lock` returns void, which the client cannot + // deserialize as a column — selecting FROM it yields a plain int row.) + await tx.$queryRaw` + SELECT 1 AS locked + FROM pg_advisory_xact_lock(hashtextextended('documents-quota:' || ${userId}, 0)) + `; + const rows = await tx.$queryRaw>` + SELECT COALESCE(SUM(byte_size), 0)::bigint AS used + FROM inbound_documents + WHERE user_id = ${userId} + `; + const usedBytes = Number(rows[0]?.used ?? 0); + if (usedBytes + bytes.byteLength > limits.quotaBytes) { + throw new QuotaExceededError(usedBytes); + } + + const { content, codec } = encryptDocumentContent(bytes); + + // No mass assignment — every column is set field-by-field; `userId` + // comes from the session, never the body. `documentDate` defaults to + // the upload day so display == sort == filter (user-editable later). + const created = await tx.inboundDocument.create({ + data: { + userId, + kind: input.kind ?? "OTHER", + // Stored plaintext on purpose (mirrors `filename`) so the list can + // ILIKE-search + ORDER BY it. It MAY hold PHI the user types; that + // is the accepted tradeoff for server-side search/sort — the + // document body stays encrypted. + title: input.title, + filename: input.filename ? input.filename.slice(0, 255) : null, + mimeType: detected.mimeType, + byteSize: bytes.byteLength, + contentEncrypted: content, + contentCodec: codec, + contentSha256, + status: "STORED", + documentDate: input.documentDate + ? isoDateToUtc(input.documentDate) + : new Date(), + sourceSystem, + sourceId, + sourceInstance, + aiReadDeferred: input.aiDeferred, + }, + omit: { contentEncrypted: true }, + }); + + if (episodeIds.length > 0) { + await linkTargets(tx, { + userId, + sourceKind: "document", + sourceId: created.id, + targetKind: "conditionEpisode", + targetIds: episodeIds, + }); + } + if (encounterIds.length > 0) { + await linkTargets(tx, { + userId, + sourceKind: "document", + sourceId: created.id, + targetKind: "encounter", + targetIds: encounterIds, + }); + } + return created; + }); + } catch (err) { + if (err instanceof QuotaExceededError) { + return { + kind: "quotaExceeded", + usedBytes: err.usedBytes, + quotaBytes: limits.quotaBytes, + }; + } + if (isP2002(err)) { + // A racing upload won one of the two partial unique indexes — the same + // source key, or the same bytes. Surface the winner exactly as the fast + // paths above would have. + if (sourceSystem && sourceId) { + const match = await findSourceKey( + userId, + sourceSystem, + sourceId, + sourceInstance, + ); + if (match) return { kind: "sourceKey", match }; + } + const winner = await prisma.inboundDocument.findFirst({ + where: { userId, contentSha256, deletedAt: null }, + omit: { contentEncrypted: true }, + }); + if (winner) return answerDuplicate(input, winner); + } + throw err; + } + + await auditLog("documents.inbound.store", { + userId, + ipAddress: input.ipAddress, + details: { + documentId: document.id, + mime: detected.mimeType, + ...(input.scoped ? { scoped: true } : {}), + ...(sourceSystem ? { sourceSystem } : {}), + }, + }); + + annotate({ + action: { name: "documents.vault.upload" }, + meta: { + documentId: document.id, + byteSize: document.byteSize, + servingClass: detected.servingClass, + linked: episodeIds.length, + linkedVisits: encounterIds.length, + scoped: input.scoped, + sourceSystem, + aiDeferred: input.aiDeferred, + }, + }); + + // Auto-index the freshly stored document for content search: enqueue a + // fire-and-forget background job (provider-first, local text-layer fallback). + // The caller never blocks on or fails because of indexing — the enqueue is + // not awaited and swallows its own errors (a missing boss or a transient + // send failure is a silent no-op). Only fresh inserts enqueue — a duplicate + // returns early above and never reaches here. + // + // `aiRead=defer` narrows it to the local text layer: an import of a whole + // archive must not turn into one provider call per file. The person reads + // them later, deliberately, from the document itself. + if (input.aiDeferred) { + void enqueueDocumentIndex(userId, document.id, { localOnly: true }); + } else { + void enqueueDocumentIndex(userId, document.id); + } + + // Render a preview thumbnail in the background too (pure local compute — no + // egress). Same fire-and-forget contract; a dropped enqueue is recoverable + // via the boot backfill. + void enqueueDocumentThumbnail(userId, document.id); + + // Summarise the freshly stored document in the background — but ONLY when the + // `documentsAutoAiRead` opt-in is ON (the job re-checks it and the egress + // consent, and no-ops otherwise) and the `documentAi` capability is open for + // this record. Storing never depends on AI: the document is stored above + // whatever the capability says, and the index job still runs its + // provider-free text-layer path. + // + // A deferred upload skips it outright and keeps `summaryState: NONE`, so + // the detail sheet offers "Generate summary" rather than a pending state. + const documentAi = input.aiDeferred + ? { available: false as const, reason: "deferred" } + : await input.documentAi(); + if (documentAi.available) { + void enqueueDocumentSummary(userId, document.id); + } else { + annotate({ + action: { name: "documents.summary.enqueueSkipped" }, + meta: { documentId: document.id, reason: documentAi.reason }, + }); + } + + return { + kind: "stored", + document, + servingClass: detected.servingClass, + linkedEpisodes: episodeIds.length, + linkedVisits: encounterIds.length, + }; +} diff --git a/src/lib/documents/source-key.ts b/src/lib/documents/source-key.ts index 0c5656f5b..f089aa3b4 100644 --- a/src/lib/documents/source-key.ts +++ b/src/lib/documents/source-key.ts @@ -11,8 +11,21 @@ * 3. `DocumentImportKey` — a key whose document the purge has removed. * * The first two resolve to a document (live → duplicate, tombstoned → - * deleted); the third only to "deleted". The upload route and the lookup - * route both ask here, so they cannot disagree about what a key means. + * deleted); the third only to "deleted". The upload route, the lookup route + * and the document picker all ask here, so they cannot disagree about what a + * key means. + * + * v1.39.3 — a key also names its instance (`sourceInstance`, the origin of + * the Paperless-ngx or Papra it came from): id 123 on one Paperless and id + * 123 on another are different documents. A null on either side is a + * wildcard, so nothing stored before the column existed is forgotten: + * + * - a key stored without an instance (a v1.39.2 import, an older script, a + * workflow that does not send one) matches a lookup for any instance of + * its system; + * - a lookup without an instance matches a key stored for any instance. + * + * Where both an exact and a wildcard match exist, the exact one wins. */ import { prisma } from "@/lib/db"; import { checkRateLimit, type RateLimitResult } from "@/lib/rate-limit"; @@ -23,22 +36,44 @@ export type SourceKeyMatch = | { state: "live"; document: SerialisableDocument } | { state: "deleted"; id: string | null }; +/** + * The instance half of a key match: the same instance or a stored key without + * one; any instance when the lookup has none. + */ +export function instanceMatch(sourceInstance: string | null): { + OR?: Array<{ sourceInstance: string | null }>; +} { + return sourceInstance === null + ? {} + : { OR: [{ sourceInstance }, { sourceInstance: null }] }; +} + +/** Exact instance first, then a key stored without one (nulls sort last). */ +const EXACT_FIRST = { sourceInstance: { sort: "asc", nulls: "last" } } as const; + export async function findSourceKey( userId: string, sourceSystem: DocumentSourceSystemValue, sourceId: string, + sourceInstance: string | null = null, ): Promise { + const key = { + userId, + sourceSystem, + sourceId, + ...instanceMatch(sourceInstance), + }; const own = await prisma.inboundDocument.findFirst({ - where: { userId, sourceSystem, sourceId }, + where: key, omit: { contentEncrypted: true }, + orderBy: EXACT_FIRST, }); const viaAlias = own ? null - : await prisma.documentSourceAlias.findUnique({ - where: { - userId_sourceSystem_sourceId: { userId, sourceSystem, sourceId }, - }, + : await prisma.documentSourceAlias.findFirst({ + where: key, select: { document: { omit: { contentEncrypted: true } } }, + orderBy: EXACT_FIRST, }); const document = own ?? viaAlias?.document ?? null; if (document) { @@ -46,10 +81,8 @@ export async function findSourceKey( ? { state: "deleted", id: document.id } : { state: "live", document }; } - const purged = await prisma.documentImportKey.findUnique({ - where: { - userId_sourceSystem_sourceId: { userId, sourceSystem, sourceId }, - }, + const purged = await prisma.documentImportKey.findFirst({ + where: key, select: { id: true }, }); return purged ? { state: "deleted", id: null } : null; @@ -74,13 +107,14 @@ export async function rememberSourceAlias( documentId: string, sourceSystem: DocumentSourceSystemValue, sourceId: string, + sourceInstance: string | null = null, ): Promise<"remembered" | "known" | "limit"> { const held = await prisma.documentSourceAlias.count({ where: { userId, documentId }, }); if (held >= MAX_SOURCE_ALIASES_PER_DOCUMENT) return "limit"; const { count } = await prisma.documentSourceAlias.createMany({ - data: [{ userId, documentId, sourceSystem, sourceId }], + data: [{ userId, documentId, sourceSystem, sourceInstance, sourceId }], skipDuplicates: true, }); return count > 0 ? "remembered" : "known"; @@ -107,3 +141,87 @@ export function checkSourceLookupRateLimit( SOURCE_LOOKUP_WINDOW_MS, ); } + +/** What the vault holds for one source key, as the picker lists it. */ +export type SourceKeyState = + { state: "imported"; documentId: string } | { state: "deleted" }; + +/** + * {@link findSourceKey} for a page of results at once: the same three places, + * three queries for the whole page instead of three per result. A key held by + * a live document answers `imported` with its id; one held by a tombstone, or + * kept by the purge ledger, answers `deleted`. A key the vault does not know + * is absent from the map. + */ +export async function findSourceKeyStates( + userId: string, + sourceSystem: DocumentSourceSystemValue, + sourceIds: string[], + sourceInstance: string | null = null, +): Promise> { + const states = new Map(); + const ids = [...new Set(sourceIds)]; + if (ids.length === 0) return states; + + const where = { + userId, + sourceSystem, + sourceId: { in: ids }, + ...instanceMatch(sourceInstance), + }; + const [own, aliases, purged] = await Promise.all([ + prisma.inboundDocument.findMany({ + where, + select: { + id: true, + sourceId: true, + sourceInstance: true, + deletedAt: true, + }, + }), + prisma.documentSourceAlias.findMany({ + where, + select: { + sourceId: true, + sourceInstance: true, + document: { select: { id: true, deletedAt: true } }, + }, + }), + prisma.documentImportKey.findMany({ + where, + select: { sourceId: true }, + }), + ]); + // A wildcard (instance-less) row first, so an exact-instance row after it + // wins, as it does in `findSourceKey`. + const wildcardFirst = ( + rows: T[], + ) => + [...rows].sort( + (a, b) => + Number(a.sourceInstance === sourceInstance) - + Number(b.sourceInstance === sourceInstance), + ); + + // Lowest precedence first, so the document's own key wins, as it does in + // `findSourceKey`. + for (const row of purged) states.set(row.sourceId, { state: "deleted" }); + for (const row of wildcardFirst(aliases)) { + states.set( + row.sourceId, + row.document.deletedAt + ? { state: "deleted" } + : { state: "imported", documentId: row.document.id }, + ); + } + for (const row of wildcardFirst(own)) { + if (!row.sourceId) continue; + states.set( + row.sourceId, + row.deletedAt + ? { state: "deleted" } + : { state: "imported", documentId: row.id }, + ); + } + return states; +} diff --git a/src/lib/documents/sources/__tests__/clients.test.ts b/src/lib/documents/sources/__tests__/clients.test.ts new file mode 100644 index 000000000..bf6850942 --- /dev/null +++ b/src/lib/documents/sources/__tests__/clients.test.ts @@ -0,0 +1,642 @@ +/** + * The Paperless-ngx and Papra adapters against recorded response shapes, with + * the network mocked at `safeFetch`. What these pin: + * + * - every request carries the listed origin as the operator-approved pin and + * forbids redirects, and a 3xx is reported, not followed; + * - the source's own query parameters (name, tag, date, page), and the local + * re-check that keeps a server ignoring a filter from widening a result; + * - the error mapping (401 → authRefused, 406 / an old X-Api-Version → + * versionTooOld, 404 → notFound, non-JSON → badResponse, a refused or dead + * host → unreachable); + * - the byte caps on JSON and on a download. + */ +import { beforeEach, describe, expect, it, vi } from "vitest"; + +const safeFetch = vi.hoisted(() => vi.fn()); + +vi.mock("@/lib/safe-fetch", async (importOriginal) => { + const actual = await importOriginal(); + return { ...actual, safeFetch }; +}); + +import { SafeFetchError } from "@/lib/safe-fetch"; + +import { sourceClient } from "../clients"; +import { + DocumentSourceError, + DocumentSourceTooLargeError, + SOURCE_JSON_MAX_BYTES, +} from "../http"; + +const PAPERLESS = { + system: "PAPERLESS" as const, + origin: "http://paperless.lan:8000", + baseUrl: "http://paperless.lan:8000/paperless", + organizationId: null, + token: "pl-secret-token", +}; + +const PAPRA = { + system: "PAPRA" as const, + origin: "https://papra.example.com", + baseUrl: "https://papra.example.com", + organizationId: "org_a1b2c3d4e5f6g7h8i9j0k1l2", + token: "ppk_secret", +}; + +function json(body: unknown, init: ResponseInit = {}): Response { + return new Response(JSON.stringify(body), { + status: 200, + headers: { "content-type": "application/json" }, + ...init, + }); +} + +/** Route mocked requests by path; anything unmatched is a 404. */ +function serve(routes: Record Response>) { + safeFetch.mockImplementation(async (target: string) => { + const url = new URL(target); + for (const [prefix, handler] of Object.entries(routes)) { + if (url.pathname.startsWith(prefix)) return handler(url); + } + return new Response("not found", { status: 404 }); + }); +} + +function calledUrls(): URL[] { + return safeFetch.mock.calls.map((c) => new URL(String(c[0]))); +} + +async function failure(promise: Promise) { + try { + await promise; + } catch (err) { + return err; + } + throw new Error("expected a failure"); +} + +beforeEach(() => { + safeFetch.mockReset(); +}); + +describe("every request is pinned to the listed origin", () => { + it("passes the origin as the operator-approved pin and forbids redirects", async () => { + serve({ "/paperless/api/documents/": () => json({ results: [] }) }); + await sourceClient(PAPERLESS).test(); + expect(safeFetch).toHaveBeenCalledTimes(1); + const [target, init, opts] = safeFetch.mock.calls[0]; + expect(String(target)).toBe( + "http://paperless.lan:8000/paperless/api/documents/?page_size=1&fields=id", + ); + expect(init.redirect).toBe("manual"); + expect(init.headers.Authorization).toBe("Token pl-secret-token"); + expect(init.headers.Accept).toBe("application/json; version=9"); + expect(opts.operatorApprovedPrivateOrigin).toBe( + "http://paperless.lan:8000", + ); + expect(opts.timeoutMs).toBeGreaterThan(0); + expect(opts.requirePublicHost).toBeUndefined(); + }); + + it("reports a redirect instead of following it", async () => { + serve({ + "/paperless/api/documents/": () => + new Response(null, { + status: 302, + headers: { location: "http://169.254.169.254/latest/meta-data" }, + }), + }); + const err = await failure(sourceClient(PAPERLESS).test()); + expect(err).toBeInstanceOf(DocumentSourceError); + expect((err as DocumentSourceError).code).toBe("redirected"); + expect(safeFetch).toHaveBeenCalledTimes(1); + }); + + it("reports a refused or dead destination as unreachable", async () => { + safeFetch.mockRejectedValue(new SafeFetchError("refused", "private_host")); + const err = await failure(sourceClient(PAPRA).test()); + expect((err as DocumentSourceError).code).toBe("unreachable"); + }); +}); + +describe("Paperless-ngx", () => { + const tags = { + results: [ + { id: 7, name: "Health" }, + { id: 9, name: "Insurance" }, + ], + next: null, + }; + + it("searches by title with tag, date and page, and names the tags", async () => { + serve({ + "/paperless/api/tags/": () => json(tags), + "/paperless/api/documents/": () => + json({ + count: 30, + next: "http://paperless.lan:8000/paperless/api/documents/?page=3", + results: [ + { + id: 41, + title: "Blood test", + created: "2025-03-04", + tags: [7], + original_file_name: "labor.pdf", + }, + // A server that ignored the tag filter must not widen the result. + { id: 42, title: "Car", created: "2025-03-05", tags: [9] }, + // Nor one that ignored the date filter. + { id: 43, title: "Old", created: "2019-01-01", tags: [7] }, + // No title: the file name without its extension. + { + id: 44, + title: " ", + created: "2025-06-01T10:00:00+02:00", + tags: [7], + original_file_name: "discharge-letter.pdf", + }, + ], + }), + }); + const result = await sourceClient(PAPERLESS).search({ + q: "blood", + tagId: "7", + from: "2025-01-01", + to: "2025-12-31", + page: 2, + }); + + const list = calledUrls().find((u) => u.pathname.endsWith("/documents/")); + expect(list?.searchParams.get("title__icontains")).toBe("blood"); + expect(list?.searchParams.get("tags__id__all")).toBe("7"); + expect(list?.searchParams.get("created__gte")).toBe("2025-01-01"); + expect(list?.searchParams.get("created__lte")).toBe("2025-12-31"); + // API version 9 made `created` a date; the `__date` lookups are gone. + expect(list?.searchParams.has("created__date__gte")).toBe(false); + expect(list?.searchParams.get("page")).toBe("2"); + expect(list?.searchParams.get("page_size")).toBe("25"); + + expect(result.hasMore).toBe(true); + expect(result.items).toEqual([ + { + sourceId: "41", + title: "Blood test", + date: "2025-03-04", + tags: ["Health"], + sizeBytes: null, + }, + { + sourceId: "44", + title: "discharge-letter", + date: "2025-06-01", + tags: ["Health"], + sizeBytes: null, + }, + ]); + }); + + it("answers a page past the end with nothing", async () => { + serve({ + "/paperless/api/tags/": () => json(tags), + "/paperless/api/documents/": () => + json({ detail: "Invalid page." }, { status: 404 }), + }); + await expect( + sourceClient(PAPERLESS).search({ + q: "", + tagId: null, + from: null, + to: null, + page: 99, + }), + ).resolves.toEqual({ items: [], hasMore: false }); + }); + + it("maps a refused token, an unsupported version and an old server", async () => { + serve({ "/paperless/api/": () => json({}, { status: 401 }) }); + expect( + ((await failure(sourceClient(PAPERLESS).test())) as DocumentSourceError) + .code, + ).toBe("authRefused"); + + serve({ "/paperless/api/": () => json({}, { status: 406 }) }); + expect( + ((await failure(sourceClient(PAPERLESS).test())) as DocumentSourceError) + .code, + ).toBe("versionTooOld"); + + serve({ + "/paperless/api/": () => + json({ results: [] }, { headers: { "x-api-version": "5" } }), + }); + expect( + ((await failure(sourceClient(PAPERLESS).test())) as DocumentSourceError) + .code, + ).toBe("versionTooOld"); + }); + + it("reports a web page where the API should be as a bad response", async () => { + serve({ + "/paperless/api/": () => + new Response("login", { + headers: { "content-type": "text/html" }, + }), + }); + const err = await failure(sourceClient(PAPERLESS).test()); + expect((err as DocumentSourceError).code).toBe("badResponse"); + }); + + it("refuses a JSON answer larger than the cap", async () => { + serve({ + "/paperless/api/": () => + new Response("x".repeat(SOURCE_JSON_MAX_BYTES + 10), { status: 200 }), + }); + const err = await failure(sourceClient(PAPERLESS).test()); + expect((err as DocumentSourceError).code).toBe("badResponse"); + }); + + it("reads a document's details and downloads the original, within the cap", async () => { + serve({ + "/paperless/api/documents/41/download/": () => + new Response(new Uint8Array([37, 80, 68, 70]), { + headers: { "content-disposition": 'attachment; filename="x.pdf"' }, + }), + "/paperless/api/documents/41/": () => + json({ + id: 41, + title: "Blood test", + created: "2025-03-04", + original_file_name: "labor.pdf", + }), + }); + const client = sourceClient(PAPERLESS); + await expect(client.document("41")).resolves.toEqual({ + title: "Blood test", + date: "2025-03-04", + filename: "labor.pdf", + }); + const file = await client.download("41", 1024); + expect([...file.bytes]).toEqual([37, 80, 68, 70]); + expect(file.filename).toBe("x.pdf"); + const download = calledUrls().at(-1); + expect(download?.searchParams.get("original")).toBe("true"); + }); + + it("refuses a download larger than the vault accepts, declared or not", async () => { + serve({ + "/paperless/api/documents/41/download/": () => + new Response(new Uint8Array(2048), { + headers: { "content-length": "2048" }, + }), + }); + expect( + await failure(sourceClient(PAPERLESS).download("41", 1024)), + ).toBeInstanceOf(DocumentSourceTooLargeError); + + serve({ + "/paperless/api/documents/41/download/": () => + new Response( + new ReadableStream({ + start(controller) { + controller.enqueue(new Uint8Array(800)); + controller.enqueue(new Uint8Array(800)); + controller.close(); + }, + }), + ), + }); + expect( + await failure(sourceClient(PAPERLESS).download("41", 1024)), + ).toBeInstanceOf(DocumentSourceTooLargeError); + }); +}); + +describe("ids never shape a request path", () => { + it("refuses a Paperless id that is not a number, and dot segments, before dialling", async () => { + const client = sourceClient(PAPERLESS); + for (const id of ["..", ".", "41/../x", "abc", "41?x=1"]) { + const err = await failure(client.document(id)); + expect((err as DocumentSourceError).code, id).toBe("notFound"); + expect( + ((await failure(client.download(id, 10))) as DocumentSourceError).code, + ).toBe("notFound"); + } + expect(safeFetch).not.toHaveBeenCalled(); + }); + + it("refuses a Papra id or organization id outside [A-Za-z0-9_-]", async () => { + for (const id of ["..", "doc/1", "doc%2F1", "doc 1"]) { + await failure(sourceClient(PAPRA).document(id)); + } + expect(() => + sourceClient({ ...PAPRA, organizationId: "../admin" }), + ).toThrow(DocumentSourceError); + expect(safeFetch).not.toHaveBeenCalled(); + }); + + it("leaves a result with an unusable id out of the list", async () => { + serve({ + "/paperless/api/tags/": () => json({ results: [], next: null }), + "/paperless/api/documents/": () => + json({ + next: null, + results: [ + { id: "..", title: "Odd", created: "2025-01-01", tags: [] }, + { id: 5, title: "Fine", created: "2025-01-01", tags: [] }, + ], + }), + }); + const result = await sourceClient(PAPERLESS).search({ + q: "", + tagId: null, + from: null, + to: null, + page: 1, + }); + expect(result.items.map((i) => i.sourceId)).toEqual(["5"]); + }); +}); + +describe("Paperless-ngx tags", () => { + it("pages through tags beyond the first 250", async () => { + serve({ + "/paperless/api/tags/": (url) => { + const page = Number(url.searchParams.get("page")); + const results = Array.from( + { length: page === 1 ? 250 : 3 }, + (_, i) => ({ + id: page * 1000 + i, + name: `t${page}-${i}`, + }), + ); + return json({ results, next: page === 1 ? "more" : null }); + }, + }); + const tags = await sourceClient(PAPERLESS).tags(); + expect(tags).toHaveLength(253); + expect(calledUrls().map((u) => u.searchParams.get("page"))).toEqual([ + "1", + "2", + ]); + }); +}); + +describe("Papra, as it answers for real", () => { + it("quotes each word so a leading dash is not a negation, and sends the date range", async () => { + serve({ + "/api/organizations/org_a1b2c3d4e5f6g7h8i9j0k1l2/documents": () => + json({ documents: [], documentsCount: 0 }), + }); + await sourceClient(PAPRA).search({ + q: "-Knie Entlass", + tagId: null, + from: "2024-01-01", + to: "2024-12-31", + page: 1, + }); + const list = calledUrls().find((u) => u.pathname.endsWith("/documents")); + expect(list?.searchParams.get("searchQuery")).toBe( + '"-Knie" "Entlass" date:>=2024-01-01 date:<=2024-12-31', + ); + }); + + it("dates a search row by its document date only, as Papra's date filter does", async () => { + serve({ + "/api/organizations/org_a1b2c3d4e5f6g7h8i9j0k1l2/documents": () => + json({ + documentsCount: 2, + documents: [ + { + id: "doc_dated", + name: "A.pdf", + documentDate: "2024-05-01", + createdAt: "2025-01-01T00:00:00.000Z", + }, + { + id: "doc_undated", + name: "B.pdf", + createdAt: "2025-01-01T00:00:00.000Z", + }, + ], + }), + }); + const all = await sourceClient(PAPRA).search({ + q: "", + tagId: null, + from: null, + to: null, + page: 1, + }); + expect(all.items.map((i) => [i.sourceId, i.date])).toEqual([ + ["doc_dated", "2024-05-01"], + ["doc_undated", null], + ]); + const ranged = await sourceClient(PAPRA).search({ + q: "", + tagId: null, + from: "2024-01-01", + to: "2025-12-31", + page: 1, + }); + // Papra's date: filter never matches a document without a document date; + // the per-row check agrees instead of re-admitting it by upload day. + expect(ranged.items.map((i) => i.sourceId)).toEqual(["doc_dated"]); + }); + + it("still files an undated Papra document under the day it was added on import", async () => { + serve({ + "/api/organizations/org_a1b2c3d4e5f6g7h8i9j0k1l2/documents/doc_undated": + () => + json({ + document: { + id: "doc_undated", + name: "B.pdf", + createdAt: "2025-01-01T09:00:00.000Z", + }, + }), + }); + await expect( + sourceClient(PAPRA).document("doc_undated"), + ).resolves.toMatchObject({ + date: "2025-01-01", + }); + }); + + it("refuses an organization id Papra would not issue", () => { + for (const org of ["org_123", "../admin", "ORG_A1B2C3D4E5F6G7H8I9J0K1L2"]) { + expect(() => sourceClient({ ...PAPRA, organizationId: org })).toThrow( + DocumentSourceError, + ); + } + expect(safeFetch).not.toHaveBeenCalled(); + }); + + it("reads 'not in this organization' and a malformed id as no such organization", async () => { + serve({ + "/api/organizations/": () => + json({ error: { code: "user.not_in_organization" } }, { status: 403 }), + }); + expect( + ((await failure(sourceClient(PAPRA).test())) as DocumentSourceError).code, + ).toBe("notFound"); + serve({ + "/api/organizations/": () => + json( + { error: { code: "server.invalid_request.params" } }, + { status: 400 }, + ), + }); + expect( + ((await failure(sourceClient(PAPRA).test())) as DocumentSourceError).code, + ).toBe("notFound"); + // Any other refusal of the key is still the key. + serve({ + "/api/organizations/": () => + json({ error: { code: "auth.unauthorized" } }, { status: 401 }), + }); + expect( + ((await failure(sourceClient(PAPRA).test())) as DocumentSourceError).code, + ).toBe("authRefused"); + }); + + it("fails the connection test when the key cannot read tags", async () => { + serve({ + "/api/organizations/org_a1b2c3d4e5f6g7h8i9j0k1l2/tags": () => + json({}, { status: 401 }), + "/api/organizations/org_a1b2c3d4e5f6g7h8i9j0k1l2/documents": () => + json({ documents: [], documentsCount: 0 }), + }); + const err = await failure(sourceClient(PAPRA).test()); + expect((err as DocumentSourceError).code).toBe("permissionMissing"); + }); +}); + +describe("Papra", () => { + const tags = { + tags: [ + { id: "tag_b", name: "Lab results" }, + { id: "tag_a", name: "Health" }, + ], + }; + + it("sends the name and the tag through searchQuery and re-checks both", async () => { + serve({ + "/api/organizations/org_a1b2c3d4e5f6g7h8i9j0k1l2/tags": () => json(tags), + "/api/organizations/org_a1b2c3d4e5f6g7h8i9j0k1l2/documents": () => + json({ + documentsCount: 60, + documents: [ + { + id: "doc_1", + name: "Blood test.pdf", + originalName: "Blood test.pdf", + originalSize: 1234, + documentDate: "2025-03-04", + createdAt: "2025-03-04T09:00:00.000Z", + tags: [{ id: "tag_b", name: "Lab results" }], + }, + { + id: "doc_2", + name: "Other.pdf", + createdAt: "2025-03-04T09:00:00.000Z", + tags: [{ id: "tag_a", name: "Health" }], + }, + { + id: "doc_3", + name: "Too old.pdf", + documentDate: "2018-01-01", + createdAt: "2025-03-04T09:00:00.000Z", + tags: [{ id: "tag_b", name: "Lab results" }], + }, + ], + }), + }); + const result = await sourceClient(PAPRA).search({ + q: "blood", + tagId: "tag_b", + from: "2025-01-01", + to: null, + page: 1, + }); + const list = calledUrls().find((u) => u.pathname.endsWith("/documents")); + expect(list?.searchParams.get("searchQuery")).toBe( + '"blood" tag:"Lab results" date:>=2025-01-01', + ); + expect(list?.searchParams.get("pageIndex")).toBe("0"); + expect(list?.searchParams.get("pageSize")).toBe("25"); + expect(safeFetch.mock.calls[0][1].headers.Authorization).toBe( + "Bearer ppk_secret", + ); + expect(result).toEqual({ + items: [ + { + sourceId: "doc_1", + title: "Blood test", + date: "2025-03-04", + tags: ["Lab results"], + sizeBytes: 1234, + }, + ], + hasMore: true, + }); + }); + + it("matches nothing for a tag that no longer exists", async () => { + serve({ + "/api/organizations/org_a1b2c3d4e5f6g7h8i9j0k1l2/tags": () => json(tags), + }); + await expect( + sourceClient(PAPRA).search({ + q: "", + tagId: "tag_gone", + from: null, + to: null, + page: 1, + }), + ).resolves.toEqual({ items: [], hasMore: false }); + expect(calledUrls().some((u) => u.pathname.endsWith("/documents"))).toBe( + false, + ); + }); + + it("lists tags by name and maps an unknown organization to notFound", async () => { + serve({ + "/api/organizations/org_a1b2c3d4e5f6g7h8i9j0k1l2/tags": () => json(tags), + }); + await expect(sourceClient(PAPRA).tags()).resolves.toEqual([ + { id: "tag_a", name: "Health" }, + { id: "tag_b", name: "Lab results" }, + ]); + + serve({}); + const err = await failure(sourceClient(PAPRA).test()); + expect((err as DocumentSourceError).code).toBe("notFound"); + }); + + it("reads a document and downloads its file", async () => { + serve({ + "/api/organizations/org_a1b2c3d4e5f6g7h8i9j0k1l2/documents/doc_1/file": + () => new Response(new Uint8Array([1, 2, 3])), + "/api/organizations/org_a1b2c3d4e5f6g7h8i9j0k1l2/documents/doc_1": () => + json({ + document: { + id: "doc_1", + name: "Blood test.pdf", + originalName: "blood.pdf", + documentDate: "2025-02-01", + createdAt: "2025-03-04T09:00:00.000Z", + }, + }), + }); + const client = sourceClient(PAPRA); + await expect(client.document("doc_1")).resolves.toEqual({ + title: "Blood test", + date: "2025-02-01", + filename: "blood.pdf", + }); + const file = await client.download("doc_1", 1024); + expect([...file.bytes]).toEqual([1, 2, 3]); + }); +}); diff --git a/src/lib/documents/sources/__tests__/origins.test.ts b/src/lib/documents/sources/__tests__/origins.test.ts new file mode 100644 index 000000000..1dace82ef --- /dev/null +++ b/src/lib/documents/sources/__tests__/origins.test.ts @@ -0,0 +1,110 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; + +import { + configuredDocumentSourceOrigins, + documentSourcesAvailable, + evaluateSourceBaseUrl, + listedDocumentSourceOrigins, +} from "../origins"; + +const ORIGINAL = process.env.DOCUMENT_SOURCE_ORIGINS; + +afterEach(() => { + if (ORIGINAL === undefined) delete process.env.DOCUMENT_SOURCE_ORIGINS; + else process.env.DOCUMENT_SOURCE_ORIGINS = ORIGINAL; + vi.restoreAllMocks(); +}); + +describe("DOCUMENT_SOURCE_ORIGINS", () => { + it("is off when unset or empty", () => { + delete process.env.DOCUMENT_SOURCE_ORIGINS; + expect(documentSourcesAvailable()).toBe(false); + process.env.DOCUMENT_SOURCE_ORIGINS = " "; + expect(documentSourcesAvailable()).toBe(false); + }); + + it("keeps valid exact origins and drops malformed ones with a redacted warning", () => { + const warn = vi.spyOn(console, "warn").mockImplementation(() => {}); + process.env.DOCUMENT_SOURCE_ORIGINS = + "https://paperless.example.com, http://papra.lan:1221, https://user:secret@bad.example/x?token=abc, http://169.254.169.254"; + const origins = configuredDocumentSourceOrigins(); + expect([...origins].sort()).toEqual([ + "http://papra.lan:1221", + "https://paperless.example.com", + ]); + expect(documentSourcesAvailable()).toBe(true); + const logged = warn.mock.calls.map((c) => String(c[0])).join("\n"); + expect(logged).toContain("DOCUMENT_SOURCE_ORIGINS"); + expect(logged).not.toContain("secret"); + expect(logged).not.toContain("token=abc"); + }); + + it("lists the origins for the settings card", () => { + process.env.DOCUMENT_SOURCE_ORIGINS = + "https://paperless.example.com,http://papra.lan:1221"; + expect(listedDocumentSourceOrigins()).toEqual([ + "http://papra.lan:1221", + "https://paperless.example.com", + ]); + }); +}); + +describe("evaluateSourceBaseUrl", () => { + const listed = new Set(["https://paperless.example.com", "http://10.0.0.5"]); + + it("accepts a listed origin and keeps a base path without the trailing slash", () => { + expect( + evaluateSourceBaseUrl("https://paperless.example.com/paperless/", listed), + ).toEqual({ + ok: true, + origin: "https://paperless.example.com", + baseUrl: "https://paperless.example.com/paperless", + }); + expect(evaluateSourceBaseUrl("http://10.0.0.5", listed)).toEqual({ + ok: true, + origin: "http://10.0.0.5", + baseUrl: "http://10.0.0.5", + }); + }); + + it("refuses an origin that is not listed, public or private", () => { + for (const url of [ + "https://example.org", + "http://10.0.0.6", + "http://192.168.1.10:8000", + "http://169.254.169.254", + "http://localhost:8000", + ]) { + expect(evaluateSourceBaseUrl(url, listed), url).toEqual({ + ok: false, + reason: "notAllowed", + }); + } + }); + + it("does not turn a listed host into a suffix, sibling port or other scheme grant", () => { + for (const url of [ + "https://evil.paperless.example.com", + "https://paperless.example.com:8443", + "http://paperless.example.com", + "https://paperless.example.com.evil.test", + ]) { + expect(evaluateSourceBaseUrl(url, listed).ok, url).toBe(false); + } + }); + + it("refuses userinfo, a query, a fragment and non-http schemes", () => { + for (const url of [ + "https://user:pass@paperless.example.com", + "https://paperless.example.com/?token=x", + "https://paperless.example.com/#x", + "ftp://paperless.example.com", + "not a url", + ]) { + expect(evaluateSourceBaseUrl(url, listed), url).toEqual({ + ok: false, + reason: "invalid", + }); + } + }); +}); diff --git a/src/lib/documents/sources/clients.ts b/src/lib/documents/sources/clients.ts new file mode 100644 index 000000000..5580e9f65 --- /dev/null +++ b/src/lib/documents/sources/clients.ts @@ -0,0 +1,533 @@ +/** + * The two source adapters the document picker speaks: Paperless-ngx and Papra. + * + * Each turns the picker's four questions (does this connection work, which + * tags exist, which documents match, give me this one) into the source's own + * REST calls, and each answer into the picker's minimal shapes. Everything + * leaves through `./http.ts`, so the origin pin, the redirect refusal, the + * timeouts and the byte caps are not something an adapter can forget. + * + * Both adapters re-check tag and date filters on what comes back. A server + * that ignores a parameter (an older Paperless, a Papra whose search syntax + * changed) must not widen a result into documents the person filtered out; + * the import script makes the same check for the same reason. + * + * API versions: Paperless-ngx is asked for API version 9 (Paperless-ngx 2.16 + * and later, the version where `created` became a plain date; it answers 406 + * to a version it does not speak). Papra's endpoints are the documented + * organization-scoped ones (`/api/organizations/{id}/documents`, `/tags`, + * `/documents/{id}`, `/documents/{id}/file`). + */ +import { DOCUMENT_TITLE_MAX } from "@/lib/validations/inbound-documents"; + +import { + DocumentSourceError, + sourceDownload, + sourceJson, + type SourceTarget, +} from "./http"; +import { + DOCUMENT_PICKER_MAX_TAGS, + DOCUMENT_PICKER_PAGE_SIZE, + isSourceDocumentId, + PAPRA_ORG_ID, + type DocumentPickerSystem, + type DocumentSourceErrorCode, + type DocumentSourceTagDto, +} from "./types"; + +/** Paperless-ngx API version the picker speaks (Paperless-ngx 2.16+). */ +export const PAPERLESS_API_VERSION = 9; + +export interface SourceSearchParams { + /** Name to search for; empty lists everything. */ + q: string; + /** A tag id as the source names it. */ + tagId: string | null; + /** YYYY-MM-DD, inclusive. */ + from: string | null; + to: string | null; + /** 1-based. */ + page: number; +} + +export interface SourceListItem { + sourceId: string; + title: string; + date: string | null; + tags: string[]; + sizeBytes: number | null; +} + +export interface SourceDocumentMeta { + title: string; + date: string | null; + filename: string | null; +} + +export interface SourceClient { + readonly system: DocumentPickerSystem; + /** + * The instance this client talks to, as the normalised origin every source + * key it produces carries (`sourceInstance`). + */ + readonly instance: string; + test(): Promise; + tags(): Promise; + search( + params: SourceSearchParams, + ): Promise<{ items: SourceListItem[]; hasMore: boolean }>; + document(sourceId: string): Promise; + download( + sourceId: string, + maxFileBytes: number, + ): Promise<{ bytes: Uint8Array; filename: string | null }>; +} + +export interface SourceConnection { + system: DocumentPickerSystem; + origin: string; + baseUrl: string; + organizationId: string | null; + token: string; +} + +/** Paperless tags are read this many per page. */ +const TAG_PAGE_SIZE = 250; + +/** Refuse an id that is not one the system issues, before it reaches a URL. */ +function requireId(system: DocumentPickerSystem, id: string): void { + if (!isSourceDocumentId(system, id)) { + throw new DocumentSourceError("notFound"); + } +} + +// ── Defensive readers ────────────────────────────────────────────────────── + +type Json = Record; + +function asRecord(value: unknown): Json | null { + return value && typeof value === "object" && !Array.isArray(value) + ? (value as Json) + : null; +} + +function asArray(value: unknown): unknown[] { + return Array.isArray(value) ? value : []; +} + +function asText(value: unknown): string | null { + if (typeof value === "string") return value; + if (typeof value === "number" && Number.isFinite(value)) return String(value); + return null; +} + +/** The body is not the shape the API documents: most often a base address + * that points at the web app instead of the instance. */ +function shapeError(): DocumentSourceError { + return new DocumentSourceError("badResponse"); +} + +/** YYYY-MM-DD from an ISO date or date-time, or null. */ +export function dayOf(value: unknown): string | null { + if (typeof value !== "string") return null; + const day = value.slice(0, 10); + return /^\d{4}-\d{2}-\d{2}$/.test(day) ? day : null; +} + +function stripExtension(name: string | null): string { + return name ? name.replace(/\.[A-Za-z0-9]{1,5}$/, "") : ""; +} + +/** + * A title that is not blank and fits the vault: the source's title, else its + * file name without the extension, else a placeholder naming the source id. + */ +export function titleOf( + title: string | null, + filename: string | null, + fallback: string, +): string { + const picked = + (title ?? "").trim() || stripExtension(filename).trim() || fallback; + return picked.slice(0, DOCUMENT_TITLE_MAX); +} + +/** Is `day` inside the inclusive range? A document without a date is kept + * only when no range was asked for. */ +export function withinRange( + day: string | null, + from: string | null, + to: string | null, +): boolean { + if (!from && !to) return true; + if (!day) return false; + if (from && day < from) return false; + if (to && day > to) return false; + return true; +} + +// ── Paperless-ngx ────────────────────────────────────────────────────────── + +function paperless(connection: SourceConnection): SourceClient { + const target: SourceTarget = { + origin: connection.origin, + baseUrl: connection.baseUrl, + headers: { + Authorization: `Token ${connection.token}`, + Accept: `application/json; version=${PAPERLESS_API_VERSION}`, + }, + }; + const download: SourceTarget = { + ...target, + headers: { Authorization: `Token ${connection.token}` }, + }; + + /** A server that says which API version it speaks, and it is too old. */ + function checkVersion(response: Response): void { + const raw = response.headers.get("x-api-version"); + if (raw === null) return; + const version = Number(raw); + if (Number.isFinite(version) && version < PAPERLESS_API_VERSION) { + throw new DocumentSourceError("versionTooOld", response.status); + } + } + + async function tagMap(): Promise> { + const tags = await listTags(); + return new Map(tags.map((tag) => [tag.id, tag.name])); + } + + async function listTags(): Promise { + // Paged by number rather than by following `next`: the link is the + // server's to write, the page number is ours. + const tags: DocumentSourceTagDto[] = []; + for (let pageNo = 1; tags.length < DOCUMENT_PICKER_MAX_TAGS; pageNo += 1) { + const query = new URLSearchParams({ + page: String(pageNo), + page_size: String(TAG_PAGE_SIZE), + ordering: "name", + }); + const { body, response } = await sourceJson( + target, + `/api/tags/?${query}`, + ); + checkVersion(response); + const page = asRecord(body); + if (!page) throw shapeError(); + for (const raw of asArray(page.results)) { + const tag = asRecord(raw); + const id = asText(tag?.id); + const name = asText(tag?.name); + if (id && name) tags.push({ id, name }); + } + if (typeof page.next !== "string" || !page.next) break; + } + return tags.slice(0, DOCUMENT_PICKER_MAX_TAGS); + } + + return { + system: "PAPERLESS", + instance: connection.origin, + + async test() { + const { body, response } = await sourceJson( + target, + "/api/documents/?page_size=1&fields=id", + ); + checkVersion(response); + if (!asRecord(body)) throw shapeError(); + }, + + tags: listTags, + + async search(params) { + const query = new URLSearchParams({ + page: String(params.page), + page_size: String(DOCUMENT_PICKER_PAGE_SIZE), + ordering: "-created", + truncate_content: "true", + fields: "id,title,created,added,tags,original_file_name", + }); + if (params.q) query.set("title__icontains", params.q); + if (params.tagId) query.set("tags__id__all", params.tagId); + // API version 9 made `created` a plain date, so the range filters on + // it directly (`created__gte` / `created__lte`), both ends inclusive. + if (params.from) query.set("created__gte", params.from); + if (params.to) query.set("created__lte", params.to); + + let page: Json | null; + let names: Map; + try { + const [list, tags] = await Promise.all([ + sourceJson(target, `/api/documents/?${query}`), + tagMap(), + ]); + checkVersion(list.response); + page = asRecord(list.body); + names = tags; + } catch (err) { + // Paperless answers a page past the end with 404 ("Invalid page"): + // there is simply nothing more. + if (err instanceof DocumentSourceError && err.code === "notFound") { + return { items: [], hasMore: false }; + } + throw err; + } + if (!page) throw shapeError(); + + const items: SourceListItem[] = []; + for (const raw of asArray(page.results)) { + const doc = asRecord(raw); + const id = asText(doc?.id); + if (!doc || !id || !isSourceDocumentId("PAPERLESS", id)) continue; + const tagIds = asArray(doc.tags) + .map(asText) + .filter((t): t is string => t !== null); + if (params.tagId && !tagIds.includes(params.tagId)) continue; + const date = dayOf(doc.created) ?? dayOf(doc.added); + if (!withinRange(date, params.from, params.to)) continue; + items.push({ + sourceId: id, + title: titleOf( + asText(doc.title), + asText(doc.original_file_name), + `Paperless ${id}`, + ), + date, + tags: tagIds + .map((tagId) => names.get(tagId)) + .filter((n): n is string => typeof n === "string"), + sizeBytes: null, + }); + } + return { items, hasMore: typeof page.next === "string" && !!page.next }; + }, + + async document(sourceId) { + requireId("PAPERLESS", sourceId); + const { body, response } = await sourceJson( + target, + `/api/documents/${encodeURIComponent(sourceId)}/?fields=id,title,created,added,original_file_name`, + ); + checkVersion(response); + const doc = asRecord(body); + if (!doc) throw shapeError(); + const filename = asText(doc.original_file_name); + return { + title: titleOf(asText(doc.title), filename, `Paperless ${sourceId}`), + date: dayOf(doc.created) ?? dayOf(doc.added), + filename, + }; + }, + + async download(sourceId, maxFileBytes) { + requireId("PAPERLESS", sourceId); + // The original, not the archived PDF/A: the same bytes a Paperless + // workflow or the import script sends, so they dedup against each other. + return sourceDownload( + download, + `/api/documents/${encodeURIComponent(sourceId)}/download/?original=true`, + maxFileBytes, + ); + }, + }; +} + +// ── Papra ────────────────────────────────────────────────────────────────── + +/** + * Papra's error bodies carry a machine code. Two of them mean "no such + * organization" rather than "wrong key": a valid organization id the key's + * user is not a member of (403 `user.not_in_organization`), and one Papra + * cannot parse (400 `server.invalid_request.params`). + */ +function papraError( + status: number, + code: string | null, +): DocumentSourceErrorCode | null { + if (status === 403 && code === "user.not_in_organization") return "notFound"; + if (status === 400 && code === "server.invalid_request.params") { + return "notFound"; + } + return null; +} + +/** Papra's search syntax quotes a tag name that holds a space. */ +function papraTagTerm(name: string): string { + return /[\s"]/.test(name) ? `tag:${JSON.stringify(name)}` : `tag:${name}`; +} + +function papra(connection: SourceConnection): SourceClient { + if ( + !connection.organizationId || + !PAPRA_ORG_ID.test(connection.organizationId) + ) { + throw new DocumentSourceError("notFound"); + } + const org = `/api/organizations/${encodeURIComponent(connection.organizationId)}`; + const target: SourceTarget = { + origin: connection.origin, + baseUrl: connection.baseUrl, + headers: { + Authorization: `Bearer ${connection.token}`, + Accept: "application/json", + }, + classify: papraError, + }; + const download: SourceTarget = { + ...target, + headers: { Authorization: `Bearer ${connection.token}` }, + }; + + async function listTags(): Promise { + const { body } = await sourceJson(target, `${org}/tags`); + const page = asRecord(body); + if (!page) throw shapeError(); + const tags: DocumentSourceTagDto[] = []; + for (const raw of asArray(page.tags)) { + const tag = asRecord(raw); + const id = asText(tag?.id); + const name = asText(tag?.name); + if (id && name) tags.push({ id, name }); + } + return tags + .sort((a, b) => a.name.localeCompare(b.name)) + .slice(0, DOCUMENT_PICKER_MAX_TAGS); + } + + return { + system: "PAPRA", + instance: connection.origin, + + async test() { + const { body } = await sourceJson( + target, + `${org}/documents?pageIndex=0&pageSize=1`, + ); + if (!asRecord(body)) throw shapeError(); + // The tag filter needs `tags:read`; a key without it would save green + // and then fail on the first search. + try { + await sourceJson(target, `${org}/tags`); + } catch (err) { + if (err instanceof DocumentSourceError && err.code === "authRefused") { + throw new DocumentSourceError( + "permissionMissing", + err.upstreamStatus, + ); + } + throw err; + } + }, + + tags: listTags, + + async search(params) { + // Each word quoted: Papra reads a leading `-` as a negation and a `:` as + // a field, and a quoted word still matches as a prefix. + const terms: string[] = params.q + .split(/\s+/) + .map((word) => word.replace(/"/g, "")) + .filter((word) => word.length > 0) + .map((word) => `"${word}"`); + if (params.tagId) { + const tag = (await listTags()).find((t) => t.id === params.tagId); + // A tag that no longer exists matches nothing, rather than silently + // dropping the filter and listing everything. + if (!tag) return { items: [], hasMore: false }; + terms.push(papraTagTerm(tag.name)); + } + // The range goes to Papra as its own `date:` filter; the check on each + // row below stays as the net for a version that reads it differently. + if (params.from) terms.push(`date:>=${params.from}`); + if (params.to) terms.push(`date:<=${params.to}`); + const pageIndex = params.page - 1; + const query = new URLSearchParams({ + pageIndex: String(pageIndex), + pageSize: String(DOCUMENT_PICKER_PAGE_SIZE), + }); + if (terms.length > 0) query.set("searchQuery", terms.join(" ")); + const { body } = await sourceJson(target, `${org}/documents?${query}`); + const page = asRecord(body); + if (!page) throw shapeError(); + + const documents = asArray(page.documents); + const items: SourceListItem[] = []; + for (const raw of documents) { + const doc = asRecord(raw); + const id = asText(doc?.id); + if (!doc || !id || !isSourceDocumentId("PAPRA", id)) continue; + const tags = asArray(doc.tags) + .map(asRecord) + .filter((t): t is Json => t !== null); + if (params.tagId && !tags.some((t) => asText(t.id) === params.tagId)) { + continue; + } + // The row shows the document date only, the field Papra's `date:` + // filter reads, so the list and the filter agree: an undated document + // shows no date and falls out of any range. The check below is the net + // for a server that ignores the filter. The upload day is used only + // on import, where a filing date is needed (see `document()`). + const date = dayOf(doc.documentDate); + if (!withinRange(date, params.from, params.to)) continue; + const size = doc.originalSize; + items.push({ + sourceId: id, + title: titleOf( + stripExtension(asText(doc.name)), + asText(doc.originalName), + `Papra ${id}`, + ), + date, + tags: tags + .map((t) => asText(t.name)) + .filter((n): n is string => n !== null), + sizeBytes: typeof size === "number" && size >= 0 ? size : null, + }); + } + const total = + typeof page.documentsCount === "number" ? page.documentsCount : null; + const hasMore = + total !== null + ? (pageIndex + 1) * DOCUMENT_PICKER_PAGE_SIZE < total + : documents.length >= DOCUMENT_PICKER_PAGE_SIZE; + return { items, hasMore }; + }, + + async document(sourceId) { + requireId("PAPRA", sourceId); + const { body } = await sourceJson( + target, + `${org}/documents/${encodeURIComponent(sourceId)}`, + ); + const doc = asRecord(asRecord(body)?.document); + if (!doc) throw shapeError(); + const filename = asText(doc.originalName) ?? asText(doc.name); + return { + title: titleOf( + stripExtension(asText(doc.name)), + filename, + `Papra ${sourceId}`, + ), + date: dayOf(doc.documentDate) ?? dayOf(doc.createdAt), + filename, + }; + }, + + async download(sourceId, maxFileBytes) { + requireId("PAPRA", sourceId); + return sourceDownload( + download, + `${org}/documents/${encodeURIComponent(sourceId)}/file`, + maxFileBytes, + ); + }, + }; +} + +export function sourceClient(connection: SourceConnection): SourceClient { + return connection.system === "PAPERLESS" + ? paperless(connection) + : papra(connection); +} diff --git a/src/lib/documents/sources/connections.ts b/src/lib/documents/sources/connections.ts new file mode 100644 index 000000000..c8f0dad2c --- /dev/null +++ b/src/lib/documents/sources/connections.ts @@ -0,0 +1,117 @@ +/** + * Reading a person's document-source connection, and the one place its token + * is decrypted. + * + * {@link toConnectionDto} is the only shape a connection leaves the server in, + * and it has no field that could carry the token: `hasToken` is the literal + * `true`. {@link openConnection} decrypts the token for exactly one request to + * the source and re-checks the stored origin against the operator's list on + * the way, so a connection whose origin was removed from + * `DOCUMENT_SOURCE_ORIGINS` answers `originNotAllowed` instead of dialling. + */ +import { decrypt } from "@/lib/crypto"; +import { prisma } from "@/lib/db"; + +import { sourceClient, type SourceClient } from "./clients"; +import { DocumentSourceError } from "./http"; +import { evaluateSourceBaseUrl } from "./origins"; +import type { + DocumentPickerSystem, + DocumentSourceConnectionDto, +} from "./types"; + +/** The columns a DTO or a request needs; never the whole row by accident. */ +const CONNECTION_SELECT = { + system: true, + baseUrl: true, + organizationId: true, + tokenEncrypted: true, + lastVerifiedAt: true, +} as const; + +export interface StoredConnection { + system: string; + baseUrl: string; + organizationId: string | null; + tokenEncrypted: string; + lastVerifiedAt: Date | null; +} + +export function toConnectionDto( + row: Omit, +): DocumentSourceConnectionDto { + return { + system: row.system as DocumentPickerSystem, + baseUrl: row.baseUrl, + organizationId: row.organizationId, + hasToken: true, + lastVerifiedAt: row.lastVerifiedAt?.toISOString() ?? null, + originAllowed: evaluateSourceBaseUrl(row.baseUrl).ok, + }; +} + +export function listConnections( + userId: string, +): Promise[]> { + return prisma.documentSourceConnection.findMany({ + where: { userId }, + select: { + system: true, + baseUrl: true, + organizationId: true, + lastVerifiedAt: true, + }, + orderBy: { system: "asc" }, + }); +} + +export function loadConnection( + userId: string, + system: DocumentPickerSystem, +): Promise { + return prisma.documentSourceConnection.findUnique({ + where: { userId_system: { userId, system } }, + select: CONNECTION_SELECT, + }); +} + +/** + * A client for one of the person's stored connections, or the reason there is + * none. Throws {@link DocumentSourceError}: `notConnected` when nothing is + * stored for the system, `originNotAllowed` when the operator no longer lists + * the stored origin. + */ +export async function openConnection( + userId: string, + system: DocumentPickerSystem, +): Promise { + const row = await loadConnection(userId, system); + if (!row) throw new DocumentSourceError("notConnected"); + return clientFor(row); +} + +/** A client for a row already in hand, with the origin re-checked. */ +export function clientFor(row: StoredConnection): SourceClient { + const verdict = evaluateSourceBaseUrl(row.baseUrl); + if (!verdict.ok) throw new DocumentSourceError("originNotAllowed"); + return sourceClient({ + system: row.system as DocumentPickerSystem, + origin: verdict.origin, + baseUrl: verdict.baseUrl, + organizationId: row.organizationId, + token: decrypt(row.tokenEncrypted), + }); +} + +/** Stamp a successful test. */ +export async function markVerified( + userId: string, + system: DocumentPickerSystem, +): Promise { + const at = new Date(); + await prisma.documentSourceConnection.updateMany({ + where: { userId, system }, + data: { lastVerifiedAt: at }, + }); + return at; +} diff --git a/src/lib/documents/sources/http.ts b/src/lib/documents/sources/http.ts new file mode 100644 index 000000000..d5daf1b25 --- /dev/null +++ b/src/lib/documents/sources/http.ts @@ -0,0 +1,250 @@ +/** + * The document picker's one way to the network. + * + * Every request to a Paperless-ngx or Papra instance goes through + * {@link sourceRequest}, which goes through `safeFetch` with the connection's + * origin as `operatorApprovedPrivateOrigin`: + * + * - the target must have exactly that origin (a path, query or `next` link + * that names another host is refused before a socket opens); + * - redirects are never followed; a 3xx is reported as `redirected`, so a + * source cannot bounce the request and its `Authorization` header to + * another host; + * - the connect goes through the pinned operator-approved dispatcher, which + * refuses link-local, metadata and unspecified addresses at dial time even + * for a listed name; + * - every call has a timeout, and every body is read with a byte cap. + * + * The origin handed in is the one the caller just re-checked against + * `DOCUMENT_SOURCE_ORIGINS`. Nothing here decides trust; it only refuses to + * go anywhere else. + */ +import { safeFetch, SafeFetchError } from "@/lib/safe-fetch"; +import { BodyTooLargeError, readBoundedBody } from "@/lib/labs/ocr-upload"; +import { annotate } from "@/lib/logging/context"; + +import type { DocumentSourceErrorCode } from "./types"; + +/** JSON bodies larger than this are not a search page; refuse them. */ +export const SOURCE_JSON_MAX_BYTES = 1024 * 1024; +export const SOURCE_JSON_TIMEOUT_MS = 10_000; +export const SOURCE_DOWNLOAD_TIMEOUT_MS = 60_000; + +/** + * A request to a source ended somewhere other than a usable answer. The code + * is what the client translates; `upstreamStatus` is the only thing about the + * source's answer that is passed on. Its body never is: an error page can + * echo the request's credentials or internal paths. + */ +export class DocumentSourceError extends Error { + readonly code: DocumentSourceErrorCode; + readonly upstreamStatus?: number; + + constructor(code: DocumentSourceErrorCode, upstreamStatus?: number) { + super(`document source: ${code}`); + this.name = "DocumentSourceError"; + this.code = code; + if (upstreamStatus !== undefined) this.upstreamStatus = upstreamStatus; + } +} + +/** The document is larger than the vault accepts; carries the cap. */ +export class DocumentSourceTooLargeError extends Error { + constructor(readonly maxFileBytes: number) { + super("document source: file larger than the vault accepts"); + this.name = "DocumentSourceTooLargeError"; + } +} + +export interface SourceTarget { + /** The listed origin, exactly as `DOCUMENT_SOURCE_ORIGINS` holds it. */ + origin: string; + /** Origin plus the optional base path, no trailing slash. */ + baseUrl: string; + headers: Record; + /** How this source's error bodies read, when they say more than a status. */ + classify?: ErrorClassifier; +} + +async function sourceRequest( + target: SourceTarget, + path: string, + timeoutMs: number, +): Promise { + const url = `${target.baseUrl}${path}`; + let response: Response; + try { + response = await safeFetch( + url, + { method: "GET", headers: target.headers, redirect: "manual" }, + { operatorApprovedPrivateOrigin: target.origin, timeoutMs }, + ); + } catch (err) { + // A refused destination, a timeout and a dead host all read the same to + // the person: the source could not be reached from here. The operator's + // wide event keeps the kind. + annotate({ + meta: { + sourceFailure: err instanceof SafeFetchError ? err.kind : "network", + }, + }); + throw new DocumentSourceError("unreachable"); + } + if (response.status >= 300 && response.status < 400) { + await discard(response); + throw new DocumentSourceError("redirected", response.status); + } + return response; +} + +async function discard(response: Response): Promise { + try { + await response.body?.cancel(); + } catch { + // nothing to release + } +} + +/** + * A source-specific reading of an error answer: given the status and the + * machine code in the body (if any), the error to report, or null for the + * generic mapping. The body itself is never passed on. + */ +export type ErrorClassifier = ( + status: number, + code: string | null, +) => DocumentSourceErrorCode | null; + +/** Longest error body read to find its machine code. */ +const ERROR_BODY_MAX_BYTES = 4 * 1024; + +/** The `code` (or `error.code`) of a JSON error body, if there is one. */ +async function errorCodeOf(response: Response): Promise { + try { + const bytes = await readBoundedBody(response.body, ERROR_BODY_MAX_BYTES); + const body = JSON.parse(new TextDecoder().decode(bytes)) as unknown; + if (!body || typeof body !== "object") return null; + const record = body as Record; + const nested = record.error as Record | undefined; + const code = + typeof record.code === "string" + ? record.code + : nested && + typeof nested === "object" && + typeof nested.code === "string" + ? nested.code + : null; + return code && code.length <= 100 ? code : null; + } catch { + return null; + } +} + +/** Map a non-2xx answer to the error the person sees. */ +export async function failFor( + response: Response, + notFound: DocumentSourceErrorCode = "notFound", + classify?: ErrorClassifier, +): Promise { + const status = response.status; + if (classify) { + const specific = classify(status, await errorCodeOf(response)); + if (specific) throw new DocumentSourceError(specific, status); + } else { + await discard(response); + } + if (status === 401 || status === 403) { + throw new DocumentSourceError("authRefused", status); + } + if (status === 404) throw new DocumentSourceError(notFound, status); + // Paperless-ngx answers 406 to an API version it no longer (or not yet) + // speaks. + if (status === 406) throw new DocumentSourceError("versionTooOld", status); + throw new DocumentSourceError("badResponse", status); +} + +/** GET a JSON document, bounded in time and size. */ +export async function sourceJson( + target: SourceTarget, + path: string, + options: { + notFound?: DocumentSourceErrorCode; + classify?: ErrorClassifier; + } = {}, +): Promise<{ body: unknown; response: Response }> { + const response = await sourceRequest(target, path, SOURCE_JSON_TIMEOUT_MS); + if (!response.ok) { + return failFor( + response, + options.notFound, + options.classify ?? target.classify, + ); + } + const declared = Number(response.headers.get("content-length") ?? 0); + if (declared > SOURCE_JSON_MAX_BYTES) { + await discard(response); + throw new DocumentSourceError("badResponse", response.status); + } + let bytes: Uint8Array; + try { + bytes = await readBoundedBody(response.body, SOURCE_JSON_MAX_BYTES); + } catch { + throw new DocumentSourceError("badResponse", response.status); + } + try { + return { + body: JSON.parse(new TextDecoder().decode(bytes)) as unknown, + response, + }; + } catch { + // A login page or a proxy's HTML error instead of the API: most often a + // base address that points at the web app rather than its API root. + throw new DocumentSourceError("badResponse", response.status); + } +} + +/** GET a file, bounded by the vault's own per-file cap. */ +export async function sourceDownload( + target: SourceTarget, + path: string, + maxFileBytes: number, +): Promise<{ bytes: Uint8Array; filename: string | null }> { + const response = await sourceRequest( + target, + path, + SOURCE_DOWNLOAD_TIMEOUT_MS, + ); + if (!response.ok) return failFor(response, "notFound", target.classify); + const declared = Number(response.headers.get("content-length") ?? 0); + if (declared > maxFileBytes) { + await discard(response); + throw new DocumentSourceTooLargeError(maxFileBytes); + } + try { + const bytes = await readBoundedBody(response.body, maxFileBytes); + return { + bytes, + filename: filenameFrom(response.headers.get("content-disposition")), + }; + } catch (err) { + if (err instanceof BodyTooLargeError) { + throw new DocumentSourceTooLargeError(maxFileBytes); + } + throw new DocumentSourceError("unreachable"); + } +} + +/** The file name a download names, if it names one. */ +export function filenameFrom(header: string | null): string | null { + if (!header) return null; + const star = /filename\*=(?:UTF-8'')?([^;]+)/i.exec(header); + if (star) { + try { + return decodeURIComponent(star[1].trim().replace(/^"|"$/g, "")); + } catch { + // fall through to the plain form + } + } + const plain = /filename="?([^";]+)"?/i.exec(header); + return plain ? plain[1] : null; +} diff --git a/src/lib/documents/sources/origins.ts b/src/lib/documents/sources/origins.ts new file mode 100644 index 000000000..6d7bb16a2 --- /dev/null +++ b/src/lib/documents/sources/origins.ts @@ -0,0 +1,95 @@ +/** + * `DOCUMENT_SOURCE_ORIGINS` (#1038) — the operator grant that lets the + * document picker reach a Paperless-ngx or Papra instance. + * + * DOCUMENT_SOURCE_ORIGINS="https://paperless.example.com,http://papra.lan:1221" + * + * The grammar is the one every private-origin list shares + * (`src/lib/private-origin-policy.ts`): exact `scheme://host[:port]` origins, + * no path, query, credentials, wildcard, CIDR or suffix. A malformed entry is + * logged once, reduced to scheme and host, and grants nothing; the valid + * entries beside it keep working. + * + * Unlike the notification list, this one is the WHOLE trust decision, not an + * exception on top of a public-host default. An origin that is not listed is + * refused even when it is a public address, and with the variable unset the + * picker does not exist: the routes answer 404 and the client renders no + * button. A person can only ever point their connection at a system the + * operator named. Every request to a listed origin still goes through + * `safeFetch` with the origin pinned, redirects forbidden, and the dial-time + * floor that refuses link-local, metadata and unspecified addresses even for + * a listed name. + * + * Read directly from `process.env` so the compose-whitelist guard sees it, + * parsed once per distinct value, and re-read on every use: an operator who + * removes an origin cuts off the stored connections that name it on their + * next request. + */ +import { + parsePrivateOrigins, + redactGrantEntry, +} from "@/lib/private-origin-policy"; + +const ENV_NAME = "DOCUMENT_SOURCE_ORIGINS"; + +let cachedRaw: string | undefined; +let cachedOrigins: ReadonlySet = new Set(); + +export function configuredDocumentSourceOrigins(): ReadonlySet { + const raw = process.env.DOCUMENT_SOURCE_ORIGINS; + if (raw === cachedRaw) return cachedOrigins; + cachedOrigins = parsePrivateOrigins(raw, (redactedEntry, reason) => { + console.warn(`${ENV_NAME}: ignoring entry "${redactedEntry}": ${reason}`); + }); + cachedRaw = raw; + return cachedOrigins; +} + +/** Is the picker switched on for this instance at all? */ +export function documentSourcesAvailable(): boolean { + return configuredDocumentSourceOrigins().size > 0; +} + +/** Longest base URL a connection may store. */ +export const DOCUMENT_SOURCE_BASE_URL_MAX = 2048; + +export type SourceBaseVerdict = + | { ok: true; origin: string; baseUrl: string } + | { ok: false; reason: "invalid" | "notAllowed" }; + +/** + * Validate a base URL a person typed against the operator's list. + * + * The base may carry a path (a Paperless served under `/paperless`); only its + * origin is compared, and it must be listed exactly. Userinfo, a query and a + * fragment are refused: none of them belongs in a base address, and each is + * where a pasted credential would sit. Trailing slashes are trimmed so the + * client can append `/api/...` without doubling one. + */ +export function evaluateSourceBaseUrl( + value: string, + origins: ReadonlySet = configuredDocumentSourceOrigins(), +): SourceBaseVerdict { + let url: URL; + try { + url = new URL(value.trim()); + } catch { + return { ok: false, reason: "invalid" }; + } + if (url.protocol !== "http:" && url.protocol !== "https:") { + return { ok: false, reason: "invalid" }; + } + if (url.username || url.password || url.search || url.hash) { + return { ok: false, reason: "invalid" }; + } + if (!origins.has(url.origin)) return { ok: false, reason: "notAllowed" }; + const path = url.pathname.replace(/\/+$/, ""); + return { ok: true, origin: url.origin, baseUrl: `${url.origin}${path}` }; +} + +/** The listed origins as the settings card may show them. */ +export function listedDocumentSourceOrigins(): string[] { + return [...configuredDocumentSourceOrigins()] + .map((origin) => redactGrantEntry(origin)) + .sort(); +} diff --git a/src/lib/documents/sources/route-support.ts b/src/lib/documents/sources/route-support.ts new file mode 100644 index 000000000..126144674 --- /dev/null +++ b/src/lib/documents/sources/route-support.ts @@ -0,0 +1,165 @@ +/** + * What every document-source route does before its own work, and how a source + * failure becomes a response. + * + * The admission rule, with {@link admitDocumentSourceCaller} as its second + * half: + * + * - the route's own `requireAuth()` with no scope: a session, or a `["*"]` token; a narrow + * token is refused, and so is any request acting on somebody else's record + * (403 `sharing.not_permitted`). A delegate, whatever their grant, and a + * guardian acting for a managed record never reach a connection: it is the + * owner's credential to a system outside HealthLog, and driving it would + * read the owner's whole archive there, which no grant covers; + * - then the transport must be the cookie. A `["*"]` Bearer (the native + * client, a script holding a login token) is refused: the picker is a + * browser surface, and a credential for another system is managed only + * from one; + * - the documents module must be on, and the operator must have listed at + * least one origin in `DOCUMENT_SOURCE_ORIGINS`. Without the list the + * routes answer 404 before reading a row. + */ +import type { AuthContext } from "@/lib/api-handler"; +import { apiError } from "@/lib/api-response"; +import { requireModuleEnabled } from "@/lib/modules/gate"; +import { checkRateLimit, type RateLimitResult } from "@/lib/rate-limit"; + +import { DocumentSourceError, DocumentSourceTooLargeError } from "./http"; +import { documentSourcesAvailable } from "./origins"; +import { + documentSourceErrorCode, + systemFromSlug, + type DocumentPickerSystem, + type DocumentSourceErrorCode, +} from "./types"; + +/** + * Admit a caller the route has already resolved with a bare `requireAuth()`. + * + * Each route calls `requireAuth()` itself, in its own module, so the sharing + * guard's scan sees every one of them as a bare, owner-only route rather than + * a helper it cannot follow. This adds the rest: a cookie transport, the + * documents module and the operator's list. The status read and the delete + * pass `needsList: false, needsModule: false`: they touch only the person's + * own row and never the network. + */ +export async function admitDocumentSourceCaller( + auth: AuthContext, + options: { needsList?: boolean; needsModule?: boolean } = {}, +): Promise { + if (auth.authMethod !== "cookie") { + return apiError( + "Document archives are managed from a browser session.", + 403, + { errorCode: "documents.sources.browserOnly" }, + ); + } + // Reading and removing one's own connection needs neither the module nor + // the list: a person who switched Documents off, or whose operator removed + // the list, must still be able to see and delete the stored token. + if (options.needsModule !== false) { + const gate = await requireModuleEnabled(auth.user.id, "inboundDocuments"); + if (!gate.enabled) return gate.response; + } + if (options.needsList !== false && !documentSourcesAvailable()) { + return sourceErrorResponse("unavailable"); + } + return null; +} + +/** The `{system}` path segment, or a 404 for anything else. */ +export async function systemParam( + params: Promise<{ system: string }>, +): Promise { + const { system } = await params; + const resolved = systemFromSlug(system); + return resolved ?? apiError("Unknown document system.", 404); +} + +const STATUS: Record = { + unavailable: 404, + notConnected: 404, + originNotAllowed: 422, + unreachable: 502, + redirected: 502, + authRefused: 502, + permissionMissing: 502, + versionTooOld: 502, + badResponse: 502, + notFound: 404, + linkTargetNotFound: 404, + rateLimited: 429, +}; + +const MESSAGE: Record = { + unavailable: "Document archives are not enabled on this server.", + notConnected: "This document archive is not connected.", + originNotAllowed: + "This address is not on the server's list of allowed document archives.", + unreachable: "The document archive could not be reached.", + redirected: + "The document archive redirected the request. Use the address it redirects to.", + authRefused: "The document archive refused the API token.", + permissionMissing: + "The API key cannot read the archive's tags. Give it the tags:read permission.", + versionTooOld: "The document archive's version is too old.", + badResponse: "The document archive gave an answer HealthLog cannot read.", + notFound: "The document archive has no such document or organization.", + linkTargetNotFound: "The record to link to was not found.", + rateLimited: "Too many requests. Try again later.", +}; + +export function sourceErrorResponse( + code: DocumentSourceErrorCode, + upstreamStatus?: number, + headers?: Record, +): Response { + return apiError(MESSAGE[code], STATUS[code], { + errorCode: documentSourceErrorCode(code), + ...(upstreamStatus !== undefined ? { upstreamStatus } : {}), + ...(headers ? { headers } : {}), + }); +} + +/** Map a thrown source failure to its response; rethrow anything else. */ +export function responseForSourceFailure(err: unknown): Response { + if (err instanceof DocumentSourceError) { + return sourceErrorResponse(err.code, err.upstreamStatus); + } + if (err instanceof DocumentSourceTooLargeError) { + return apiError("File is too large.", 413, { + errorCode: "documents.inbound.fileTooLarge", + reason: "fileTooLarge", + maxFileBytes: err.maxFileBytes, + }); + } + throw err; +} + +/** Saves and tests share one allowance: each one dials the source. */ +const CONNECT_LIMIT_PER_MINUTE = 10; + +export function checkConnectRateLimit( + userId: string, +): Promise { + return checkRateLimit( + `documents-source-connect:${userId}`, + CONNECT_LIMIT_PER_MINUTE, + 60_000, + ); +} + +/** + * Searches and tag reads per minute. Each one is a request to the source; a + * debounced search field sends one per pause in typing, which this leaves + * ample room for. + */ +const SEARCH_LIMIT_PER_MINUTE = 60; + +export function checkSearchRateLimit(userId: string): Promise { + return checkRateLimit( + `documents-source-search:${userId}`, + SEARCH_LIMIT_PER_MINUTE, + 60_000, + ); +} diff --git a/src/lib/documents/sources/types.ts b/src/lib/documents/sources/types.ts new file mode 100644 index 000000000..3fdbfd1aa --- /dev/null +++ b/src/lib/documents/sources/types.ts @@ -0,0 +1,157 @@ +/** + * Shared vocabulary of the document picker (#1038): which systems it speaks, + * the minimal shapes it hands to the client, and the errors a source can end + * in. A zero-dependency leaf so the client components, the OpenAPI registry + * and the routes import one list. + */ + +/** The systems the picker can search and import from. */ +export const DOCUMENT_PICKER_SYSTEMS = ["PAPERLESS", "PAPRA"] as const; +export type DocumentPickerSystem = (typeof DOCUMENT_PICKER_SYSTEMS)[number]; + +/** The lowercase path segment for each system (`/api/documents/sources/{slug}`). */ +export const DOCUMENT_PICKER_SLUGS = { + paperless: "PAPERLESS", + papra: "PAPRA", +} as const satisfies Record; +export type DocumentPickerSlug = keyof typeof DOCUMENT_PICKER_SLUGS; + +export function systemFromSlug( + slug: string | undefined, +): DocumentPickerSystem | null { + return slug && Object.hasOwn(DOCUMENT_PICKER_SLUGS, slug) + ? DOCUMENT_PICKER_SLUGS[slug as DocumentPickerSlug] + : null; +} + +export function slugForSystem( + system: DocumentPickerSystem, +): DocumentPickerSlug { + return system === "PAPERLESS" ? "paperless" : "papra"; +} + +/** Results per search page. */ +export const DOCUMENT_PICKER_PAGE_SIZE = 25; +/** Highest page a search may ask for. */ +export const DOCUMENT_PICKER_MAX_PAGE = 500; +/** Longest name query. */ +export const DOCUMENT_PICKER_QUERY_MAX = 200; +/** Documents the sheet imports in one run. */ +export const DOCUMENT_PICKER_MAX_SELECTION = 25; +/** Tags offered in the filter (read in pages of 250). */ +export const DOCUMENT_PICKER_MAX_TAGS = 1000; + +/** Paperless-ngx document ids are integers. */ +export const PAPERLESS_ID = /^\d{1,18}$/; +/** Papra document ids: letters, digits, `_` and `-`. */ +export const PAPRA_ID = /^[A-Za-z0-9_-]{1,128}$/; +/** Papra organization ids, as Papra issues them. */ +export const PAPRA_ORG_ID = /^org_[a-z0-9]{24}$/; + +/** + * Is `id` a document id `system` could have issued? Checked before an id is + * put into a request path, so nothing a request carries can add a path + * segment, a query or a `..` to the URL built from it. + */ +export function isSourceDocumentId( + system: DocumentPickerSystem, + id: string, +): boolean { + if (id === "." || id === "..") return false; + return (system === "PAPERLESS" ? PAPERLESS_ID : PAPRA_ID).test(id); +} + +/** A connection as every route may show it: never the token. */ +export interface DocumentSourceConnectionDto { + system: DocumentPickerSystem; + baseUrl: string; + organizationId: string | null; + hasToken: true; + lastVerifiedAt: string | null; + /** False once the operator removed the origin from the list. */ + originAllowed: boolean; +} + +export interface DocumentSourcesStatusDto { + /** The operator listed at least one origin; otherwise the picker is off. */ + available: boolean; + /** The listed origins, so the settings card can say what is allowed. */ + allowedOrigins: string[]; + connections: DocumentSourceConnectionDto[]; +} + +export interface DocumentSourceTagDto { + id: string; + name: string; +} + +/** Whether HealthLog already holds a result, by its source key. */ +export type DocumentPickerResultState = "new" | "imported" | "deleted"; + +export interface DocumentSourceResultDto { + sourceId: string; + title: string; + /** YYYY-MM-DD, the document's own date in the source, or null. */ + date: string | null; + tags: string[]; + sizeBytes: number | null; + state: DocumentPickerResultState; + /** The vault document when `state` is `imported`. */ + documentId: string | null; +} + +export interface DocumentSourceSearchDto { + results: DocumentSourceResultDto[]; + page: number; + hasMore: boolean; +} + +export type DocumentImportOutcome = "imported" | "duplicate" | "deleted"; + +export interface DocumentSourceImportDto { + outcome: DocumentImportOutcome; + /** The vault document; null only when a deleted one has been purged. */ + documentId: string | null; + /** The document was linked to the record the picker was opened from. */ + linked: boolean; +} + +/** Record kinds a picked document can be linked to on import. */ +export const DOCUMENT_PICKER_LINK_KINDS = [ + "conditionEpisode", + "encounter", + "vaccination", +] as const; +export type DocumentPickerLinkKind = + (typeof DOCUMENT_PICKER_LINK_KINDS)[number]; + +/** + * Why a request to a source failed, as the `meta.errorCode` values the routes + * emit. The route helper keys its status and wording on the last segment. + */ +export const DOCUMENT_SOURCE_ERROR_CODES = [ + "documents.sources.unavailable", + "documents.sources.notConnected", + "documents.sources.originNotAllowed", + "documents.sources.unreachable", + "documents.sources.redirected", + "documents.sources.authRefused", + "documents.sources.permissionMissing", + "documents.sources.versionTooOld", + "documents.sources.badResponse", + "documents.sources.notFound", + "documents.sources.linkTargetNotFound", + "documents.sources.rateLimited", +] as const; +type SourceErrorSuffix = T extends `documents.sources.${infer S}` + ? S + : never; +export type DocumentSourceErrorCode = SourceErrorSuffix< + (typeof DOCUMENT_SOURCE_ERROR_CODES)[number] +>; + +export function documentSourceErrorCode( + code: DocumentSourceErrorCode, +): (typeof DOCUMENT_SOURCE_ERROR_CODES)[number] { + return `documents.sources.${code}`; +} diff --git a/src/lib/export/backup-plan.ts b/src/lib/export/backup-plan.ts index c9fa7b80a..2d689337a 100644 --- a/src/lib/export/backup-plan.ts +++ b/src/lib/export/backup-plan.ts @@ -620,6 +620,8 @@ export const NOT_IN_BACKUP_MODELS: Readonly> = { "A CSRF nonce for a handshake that is valid for minutes and is already over by the time any backup is read.", McpOAuthConnection: "A grant issued to a client that was registered against this instance.", + DocumentSourceConnection: + "A person's API token for their Paperless-ngx or Papra instance. A backup file is not a credential store, same reasoning as ApiToken; and restored on another host it would point at an address that host's operator never allowed in DOCUMENT_SOURCE_ORIGINS. The documents imported through it are carried with the vault; reconnecting takes one form.", IntegrationStatus: "A ledger of what this deployment's syncs did. Restoring it onto another host would assert a sync history that host never had.", diff --git a/src/lib/export/records-backup.ts b/src/lib/export/records-backup.ts index d924c8c59..2bb4cd278 100644 --- a/src/lib/export/records-backup.ts +++ b/src/lib/export/records-backup.ts @@ -177,6 +177,8 @@ export interface DocumentBackupEntry { */ sourceSystem?: string | null; sourceId?: string | null; + /** v1.39.3 — the source instance (origin); absent in older files. */ + sourceInstance?: string | null; /** Canonical DR only: the import's hold on automatic AI reading. */ aiReadDeferred?: boolean; } @@ -309,6 +311,7 @@ export async function buildRecordsBackupSection( lastIndexOutcome: true, sourceSystem: true, sourceId: true, + sourceInstance: true, aiReadDeferred: true, createdAt: true, updatedAt: true, @@ -330,6 +333,7 @@ export async function buildRecordsBackupSection( summaryEncrypted: true, sourceSystem: true, sourceId: true, + sourceInstance: true, createdAt: true, }, }); @@ -545,6 +549,7 @@ export async function buildRecordsBackupSection( : null, sourceSystem: d.sourceSystem, sourceId: d.sourceId, + sourceInstance: d.sourceInstance, createdAt: d.createdAt.toISOString(), }; diff --git a/src/lib/export/restore-backup.ts b/src/lib/export/restore-backup.ts index 3e60c6b51..d10100e78 100644 --- a/src/lib/export/restore-backup.ts +++ b/src/lib/export/restore-backup.ts @@ -30,6 +30,7 @@ import { Buffer } from "node:buffer"; import { prisma, toJson } from "@/lib/db"; +import { normaliseSourceInstance } from "@/lib/validations/inbound-documents"; import { auditLog } from "@/lib/auth/audit"; import { isStoredBackupReadError, @@ -2057,6 +2058,14 @@ export async function restoreBackup( sourceId: document.sourceSystem ? (document.sourceId ?? null) : null, + // A file written before v1.39.3 has no instance: the key then + // matches any instance of its system, as it did when stored. + sourceInstance: + document.sourceSystem && document.sourceId + ? document.sourceInstance + ? normaliseSourceInstance(document.sourceInstance) + : null + : null, createdAt: new Date(document.createdAt!), updatedAt: new Date(document.updatedAt!), })), diff --git a/src/lib/jobs/document-purge.ts b/src/lib/jobs/document-purge.ts index 40106a370..d98e9a988 100644 --- a/src/lib/jobs/document-purge.ts +++ b/src/lib/jobs/document-purge.ts @@ -58,9 +58,12 @@ export async function purgeExpiredDocumentTombstones( userId: true, sourceSystem: true, sourceId: true, + sourceInstance: true, // Further keys the import answered with this document (same bytes). // They cascade away with the row, so they are copied here first. - sourceAliases: { select: { sourceSystem: true, sourceId: true } }, + sourceAliases: { + select: { sourceSystem: true, sourceId: true, sourceInstance: true }, + }, }, }); const keys = keyed.flatMap((row) => [ @@ -70,6 +73,7 @@ export async function purgeExpiredDocumentTombstones( userId: row.userId, sourceSystem: row.sourceSystem, sourceId: row.sourceId, + sourceInstance: row.sourceInstance, }, ] : []), @@ -77,6 +81,7 @@ export async function purgeExpiredDocumentTombstones( userId: row.userId, sourceSystem: alias.sourceSystem, sourceId: alias.sourceId, + sourceInstance: alias.sourceInstance, })), ]); if (keys.length > 0) { diff --git a/src/lib/openapi/error-codes.ts b/src/lib/openapi/error-codes.ts index 92602141d..bb300d4ae 100644 --- a/src/lib/openapi/error-codes.ts +++ b/src/lib/openapi/error-codes.ts @@ -136,6 +136,22 @@ export const ERROR_CODE_CATALOGUE: Readonly> = "documents.inbound.uploadBusy", "documents.inbound.uploadTimeout", "documents.inbound.vaccinationNotFound", + "documents.sources.authRefused", + "documents.sources.badResponse", + "documents.sources.browserOnly", + "documents.sources.invalidAddress", + "documents.sources.linkTargetNotFound", + "documents.sources.notConnected", + "documents.sources.notFound", + "documents.sources.organizationRequired", + "documents.sources.originNotAllowed", + "documents.sources.permissionMissing", + "documents.sources.rateLimited", + "documents.sources.redirected", + "documents.sources.tokenRequired", + "documents.sources.unavailable", + "documents.sources.unreachable", + "documents.sources.versionTooOld", ], encounter: [ "encounter.invalid", diff --git a/src/lib/openapi/routes/document-sources.ts b/src/lib/openapi/routes/document-sources.ts new file mode 100644 index 000000000..0b97d283c --- /dev/null +++ b/src/lib/openapi/routes/document-sources.ts @@ -0,0 +1,402 @@ +/** + * OpenAPI route table for the document picker (`/api/documents/sources/*`, + * #1038): a person's connection to Paperless-ngx or Papra, searching it, and + * importing picked documents into the vault. + * + * Every operation is cookie-only (declared per operation as `cookieAuth`): a + * Bearer, even a `["*"]` one, is refused 403 `documents.sources.browserOnly`, + * and so is a request acting on another person's record. The native client does + * not use these routes. Request bodies reuse the runtime Zod schemas. + */ +import type { ZodOpenApiObject } from "zod-openapi"; +import { z } from "zod/v4"; + +import { + DOCUMENT_PICKER_LINK_KINDS, + DOCUMENT_PICKER_MAX_PAGE, + DOCUMENT_PICKER_QUERY_MAX, + DOCUMENT_PICKER_SYSTEMS, +} from "@/lib/documents/sources/types"; +import { + documentSourceImportSchema, + documentSourceSaveSchema, +} from "@/lib/validations/document-sources"; + +import { dataEnvelope, errorEnvelope, stdResponses } from "./shared"; + +const cookieOnly = [{ cookieAuth: [] }]; + +const systemParameter = { + name: "system", + in: "path" as const, + required: true, + description: "`paperless` (Paperless-ngx) or `papra` (Papra).", + schema: { type: "string" as const, enum: ["paperless", "papra"] }, +}; + +const systemEnum = z.enum(DOCUMENT_PICKER_SYSTEMS); + +const connectionSchema = z + .object({ + system: systemEnum, + baseUrl: z.string(), + organizationId: z.string().nullable(), + hasToken: z.literal(true), + lastVerifiedAt: z.string().nullable(), + originAllowed: z.boolean(), + }) + .meta({ id: "DocumentSourceConnection" }); + +const statusSchema = z + .object({ + available: z.boolean(), + allowedOrigins: z.array(z.string()), + connections: z.array(connectionSchema), + }) + .meta({ id: "DocumentSourcesStatus" }); + +const tagSchema = z + .object({ id: z.string(), name: z.string() }) + .meta({ id: "DocumentSourceTag" }); + +const resultSchema = z + .object({ + sourceId: z.string(), + title: z.string(), + date: z.string().nullable(), + tags: z.array(z.string()), + sizeBytes: z.number().int().nullable(), + state: z.enum(["new", "imported", "deleted"]), + documentId: z.string().nullable(), + }) + .meta({ id: "DocumentSourceResult" }); + +const searchSchema = z + .object({ + results: z.array(resultSchema), + page: z.number().int(), + hasMore: z.boolean(), + }) + .meta({ id: "DocumentSourceSearch" }); + +const importResultSchema = z + .object({ + outcome: z.enum(["imported", "duplicate", "deleted"]), + documentId: z.string().nullable(), + linked: z.boolean(), + }) + .meta({ id: "DocumentSourceImport" }); + +const refusal = { + description: + "Refused before any request to the source: `documents.sources.browserOnly` for a Bearer caller, `sharing.not_permitted` while acting on another person's record (a delegate or guardian never uses the owner's archive), or `module.disabled` when the Documents module is off.", + content: { "application/json": { schema: errorEnvelope } }, +}; + +const unavailable = { + description: + "`documents.sources.unavailable`: the operator has not listed any origin in `DOCUMENT_SOURCE_ORIGINS`, so the picker is off. `documents.sources.notConnected`: no connection is saved for this system. `documents.sources.notFound`: the source has no such document, or (Papra) no such organization, including one the key's user is not a member of (Papra 403 `user.not_in_organization`) and one Papra cannot parse (400).", + content: { "application/json": { schema: errorEnvelope } }, +}; + +const upstream = { + description: + "The source could not answer usefully, with the reason in `meta.errorCode` and, when the source answered at all, its HTTP status in `meta.upstreamStatus` (its body is never passed on): `documents.sources.unreachable` (no connection, timeout, or a destination refused at dial time), `documents.sources.redirected` (the source answered with a redirect, which is never followed), `documents.sources.authRefused` (401/403 from the source), `documents.sources.permissionMissing` (Papra: the key cannot read tags; needs `tags:read`), `documents.sources.versionTooOld` (Paperless-ngx older than 2.16 / API version 9), `documents.sources.badResponse` (not the documented JSON, or larger than 1 MiB).", + content: { "application/json": { schema: errorEnvelope } }, +}; + +const originRefusal = { + description: + "`documents.sources.originNotAllowed`: the base address's origin is not on the operator's list (checked on every request, so a connection saved earlier is cut off once its origin is removed), or another validation failure.", + content: { "application/json": { schema: errorEnvelope } }, +}; + +export const documentSourcePaths: NonNullable = { + "/api/documents/sources": { + get: { + tags: ["Documents"], + summary: "Document archives: availability and connections", + description: + "Whether the operator enabled the document picker (`available`, true when `DOCUMENT_SOURCE_ORIGINS` lists at least one origin), the listed origins, and the caller's own saved connections to Paperless-ngx or Papra. A connection never carries its token: `hasToken` is always true for a saved one, and no route returns the token in whole or in part. `originAllowed` is false once the operator removed the connection's origin from the list. No request leaves the server. Needs neither the Documents module nor the list: with the list unset, `available` is false and any connection still stored is listed (with `originAllowed: false`) so it can be removed. Cookie session only; not delegable.", + security: cookieOnly, + responses: { + "200": { + description: "The picker's state for the caller.", + content: { + "application/json": { + schema: dataEnvelope( + statusSchema, + "DocumentSourcesStatusEnvelope", + ), + }, + }, + }, + "403": refusal, + ...stdResponses, + }, + }, + }, + "/api/documents/sources/{system}": { + put: { + tags: ["Documents"], + summary: "Save a connection to a document archive", + description: + "Saves or replaces the caller's connection to Paperless-ngx (`paperless`: base address and API token) or Papra (`papra`: base address, organization id, API key with `documents:read` and `tags:read`). The base address may carry a path; its origin must be listed in `DOCUMENT_SOURCE_ORIGINS`. The connection is tested live before anything is stored. Leaving out `token` keeps the stored one, but only when the new address is on the same origin as the saved one: the first save and a move to another origin must carry it (422 `documents.sources.tokenRequired`), so a stored token is never sent to an address it was not saved for. A Papra organization id is letters, digits, `_` and `-`. The token is stored encrypted (AES-256-GCM) and never returned. Ten saves and tests a minute per person. Cookie session only; not delegable.", + security: cookieOnly, + parameters: [systemParameter], + requestBody: { + required: true, + content: { "application/json": { schema: documentSourceSaveSchema } }, + }, + responses: { + "200": { + description: "Saved and verified. The connection, without its token.", + content: { + "application/json": { + schema: dataEnvelope( + connectionSchema, + "DocumentSourceConnectionEnvelope", + ), + }, + }, + }, + "403": refusal, + "404": unavailable, + "422": { + description: + "`documents.sources.originNotAllowed`, `documents.sources.invalidAddress`, `documents.sources.organizationRequired` (Papra), `documents.sources.tokenRequired` (first save without a token), or body validation.", + content: { "application/json": { schema: errorEnvelope } }, + }, + "502": upstream, + "401": stdResponses["401"], + "429": stdResponses["429"], + }, + }, + delete: { + tags: ["Documents"], + summary: "Remove a connection to a document archive", + description: + "Deletes the caller's connection for this system, and its token with it. Works even after the operator removed the origin from the list, and with the Documents module switched off. `disconnected` is false when there was nothing to delete. Cookie session only; not delegable.", + security: cookieOnly, + parameters: [systemParameter], + responses: { + "200": { + description: "Removed, or there was nothing to remove.", + content: { + "application/json": { + schema: dataEnvelope( + z + .object({ disconnected: z.boolean() }) + .meta({ id: "DocumentSourceDisconnect" }), + "DocumentSourceDisconnectEnvelope", + ), + }, + }, + }, + "403": refusal, + ...stdResponses, + }, + }, + }, + "/api/documents/sources/{system}/test": { + post: { + tags: ["Documents"], + summary: "Test a saved connection", + description: + "One request to the source with the stored token. Answers `ok` with the round-trip time and stamps `lastVerifiedAt`, or the reason in `meta.errorCode`. Shares the save's allowance of ten a minute. Cookie session only; not delegable.", + security: cookieOnly, + parameters: [systemParameter], + responses: { + "200": { + description: "The source answered.", + content: { + "application/json": { + schema: dataEnvelope( + z + .object({ + ok: z.literal(true), + latencyMs: z.number().int(), + lastVerifiedAt: z.string(), + }) + .meta({ id: "DocumentSourceTest" }), + "DocumentSourceTestEnvelope", + ), + }, + }, + }, + "403": refusal, + "404": unavailable, + "422": originRefusal, + "502": upstream, + "401": stdResponses["401"], + "429": stdResponses["429"], + }, + }, + }, + "/api/documents/sources/{system}/tags": { + get: { + tags: ["Documents"], + summary: "Tags of a connected archive", + description: + "The source's tags for the picker's filter, the first 250 by name, in the source's own ids. Shares the search allowance of sixty a minute. Cookie session only; not delegable.", + security: cookieOnly, + parameters: [systemParameter], + responses: { + "200": { + description: "The tags.", + content: { + "application/json": { + schema: dataEnvelope( + z + .object({ tags: z.array(tagSchema) }) + .meta({ id: "DocumentSourceTags" }), + "DocumentSourceTagsEnvelope", + ), + }, + }, + }, + "403": refusal, + "404": unavailable, + "422": originRefusal, + "502": upstream, + "401": stdResponses["401"], + "429": stdResponses["429"], + }, + }, + }, + "/api/documents/sources/{system}/search": { + get: { + tags: ["Documents"], + summary: "Search a connected archive by name, tag and date", + description: + "Searches the source 25 documents at a time. `q` matches the document's name (Paperless-ngx title, Papra name); `tag` narrows to one of the source's tags by id; `from` / `to` to a date range, inclusive (Paperless-ngx filters on `created__gte` / `created__lte`; Papra's range is applied to each page after it arrives). When a filter applied after arrival empties a page, the route reads on, up to five source pages, and `page` in the answer is the last source page read; ask for `page + 1` next. Each result carries what the vault already holds under that source key: `new`, `imported` with the vault document's `documentId`, or `deleted` (deleted in HealthLog; importing it again stores nothing). Nothing is stored. Sixty searches a minute, shared with the tag read. Cookie session only; not delegable.", + security: cookieOnly, + parameters: [ + systemParameter, + { + name: "q", + in: "query", + required: false, + schema: { type: "string", maxLength: DOCUMENT_PICKER_QUERY_MAX }, + }, + { + name: "tag", + in: "query", + required: false, + schema: { type: "string", maxLength: 64 }, + }, + { + name: "from", + in: "query", + required: false, + schema: { type: "string", format: "date" }, + }, + { + name: "to", + in: "query", + required: false, + schema: { type: "string", format: "date" }, + }, + { + name: "page", + in: "query", + required: false, + schema: { + type: "integer", + minimum: 1, + maximum: DOCUMENT_PICKER_MAX_PAGE, + default: 1, + }, + }, + ], + responses: { + "200": { + description: "One page of results.", + content: { + "application/json": { + schema: dataEnvelope( + searchSchema, + "DocumentSourceSearchEnvelope", + ), + }, + }, + }, + "403": refusal, + "404": unavailable, + "422": originRefusal, + "502": upstream, + "401": stdResponses["401"], + "429": stdResponses["429"], + }, + }, + }, + "/api/documents/sources/{system}/import": { + post: { + tags: ["Documents"], + summary: "Import one picked document", + description: + "`sourceId` must be an id the system issues (Paperless-ngx: digits; Papra: letters, digits, `_`, `-`), else 422 before anything is fetched. Fetches one document from the source and stores it in the vault through the same path as an upload, with `sourceSystem` / `sourceId` / `sourceInstance` (the connection's origin) provenance; the same id from another instance is another document. One import per key runs at a time; a concurrent second one waits and then answers `duplicate`. The key check counts toward the lookup allowance, the fetch toward the search allowance, and only a file that arrived toward the upload allowance. A source key the vault already holds is answered without a download: `duplicate` for a live document (same id), `deleted` for one the person deleted in HealthLog, which is never stored again. Bytes already in the vault under another key are `duplicate` too, and the key is remembered for that document. `link` files the document against the record the picker was opened from (a condition, a visit or a vaccination, which must be the caller's own and live; checked before anything is fetched), including a duplicate. `kind` sets the document type (default OTHER). Automatic AI reading follows the person's own setting. Counts against the person's upload allowance (60 an hour) and storage quota. 201 for a new document, 200 otherwise. Cookie session only; not delegable.", + security: cookieOnly, + parameters: [systemParameter], + requestBody: { + required: true, + content: { + "application/json": { schema: documentSourceImportSchema }, + }, + }, + responses: { + "201": { + description: "Imported.", + content: { + "application/json": { + schema: dataEnvelope( + importResultSchema, + "DocumentSourceImportEnvelope", + ), + }, + }, + }, + "200": { + description: + "Already held (`duplicate`) or deleted in HealthLog (`deleted`).", + content: { + "application/json": { + schema: dataEnvelope( + importResultSchema, + "DocumentSourceImportHeldEnvelope", + ), + }, + }, + }, + "403": refusal, + "404": { + description: `${unavailable.description} \`documents.sources.linkTargetNotFound\`: the record in \`link\` is not the caller's own live ${DOCUMENT_PICKER_LINK_KINDS.join(" / ")}.`, + content: { "application/json": { schema: errorEnvelope } }, + }, + "409": { + description: + "`documents.inbound.sourceAliasLimit`: these bytes are already stored under too many source ids.", + content: { "application/json": { schema: errorEnvelope } }, + }, + "413": { + description: + "`documents.inbound.fileTooLarge` (with `maxFileBytes`) or `documents.inbound.quotaExceeded` (with `quotaBytes` and `usedBytes`).", + content: { "application/json": { schema: errorEnvelope } }, + }, + "415": { + description: + "`documents.inbound.fileType`: the file is not a type the vault accepts.", + content: { "application/json": { schema: errorEnvelope } }, + }, + "422": originRefusal, + "502": upstream, + "401": stdResponses["401"], + "429": { + description: + "`documents.inbound.rateLimited` (the person's hourly upload allowance, with the rate-limit headers) or `documents.inbound.uploadBusy` (too many uploads in progress; `Retry-After: 1`).", + content: { "application/json": { schema: errorEnvelope } }, + }, + }, + }, + }, +}; diff --git a/src/lib/openapi/routes/documents.ts b/src/lib/openapi/routes/documents.ts index d31ad375d..9bdbcf89e 100644 --- a/src/lib/openapi/routes/documents.ts +++ b/src/lib/openapi/routes/documents.ts @@ -24,6 +24,7 @@ import { inboundConfirmSchema, inboundFactEditSchema, DOCUMENT_SOURCE_ID_MAX, + DOCUMENT_SOURCE_INSTANCE_MAX, DOCUMENT_SOURCE_SYSTEMS, DOCUMENT_SUMMARY_STATES, INBOUND_DOCUMENT_KINDS, @@ -417,12 +418,20 @@ export const inboundDocumentPaths: NonNullable = { description: "Import source key, second half. Both halves or neither; half a key is 422.", }, + { + name: "sourceInstance", + in: "query", + required: false, + schema: { type: "string", maxLength: DOCUMENT_SOURCE_INSTANCE_MAX }, + description: + "Optional, with a key. The instance of `sourceSystem` the id belongs to, as its http(s) address; normalised to the origin (`scheme://host[:port]`). The same id in two instances is two documents. Optional: a key stored without an instance (every upload before v1.39.3) matches any instance of its system, and a key sent without one matches any stored instance. If also sent in the form it must agree.", + }, ], tags: ["Documents"], summary: "Store a document (no extraction)", description: 'STORE-ONLY upload. Stores the raw document ENCRYPTED at rest with `status: STORED` and runs NO extraction — provider-free, no AI consent / budget / egress. A file is always filable, even with no document-scan provider configured. `multipart/form-data`: a `file` plus optional `title`, `kind`, `documentDate` (YYYY-MM-DD), and repeated `episodeIds` form fields (pre-link to the caller\'s illness/condition episodes). Accepted types (magic-byte sniffed, never the wire Content-Type): PDF/JPEG/PNG/WebP/GIF render inline; Office (docx/xlsx/pptx/doc/xls/ppt), text/CSV/Markdown/RTF, TIFF, HEIC/HEIF, XML/JSON are stored verbatim and served download-only. HEIC is stored as-is but attachment-only — prefer transcoding to JPEG client-side for inline preview parity. Error contract: `413` with `meta.reason = "fileTooLarge"` (+ `maxFileBytes`) or `"quotaExceeded"` (+ `quotaBytes`, `usedBytes`); `415` with `meta.reason = "unsupportedType"`. A same-user duplicate (same bytes) returns 200 + `meta.duplicate: true` with the existing row — not an error. `Idempotency-Key` honoured. Read `GET /api/documents/inbound/usage` for the effective limits before offering an upload. AI extraction is a separate opt-in action — see `POST /api/documents/inbound/{id}/extract`.\n\n' + - "**Importing from another system (v1.39.2).** Optional `sourceSystem` (`PAPERLESS`, `PAPRA`, `OTHER`) and `sourceId` (its id there, printable, up to 128 characters; needs `sourceSystem`) key the upload. A re-send with a known key is a duplicate: 200 with the live row (`meta.duplicate: true`), or, when the owner DELETED that document, 200 with `data.deleted: true` and `meta.deleted: true` and nothing stored; this holds after the 30-day purge too. The key may also ride the query string (`?sourceSystem=…&sourceId=…`): a key the vault already holds is then answered at once, the file is not stored or processed, and the request counts only toward the lookup allowance shared with `GET /api/documents/inbound/source` (5000 an hour per token, per person for a session; 429 past it), not toward the hourly upload limit. A key in both places must agree (else 422). Every other upload counts toward the hourly upload limit, duplicates included. A key that is answered with an existing document by its bytes is remembered for that document, so deleting it keeps that key deleted too; one document takes at most 20 such keys, beyond which the upload is refused 409 (`documents.inbound.sourceAliasLimit`). `OTHER` is one shared namespace per account: two different systems both sent as `OTHER` must not reuse each other\'s ids. `aiRead=defer` holds back automatic AI reading for this upload: the thumbnail and a local text index still run, the summary and lab staging do not, and the document stays out of every automatic AI path (the summary catch-up, the automatic index, Index all for search) until the person reads it with AI or generates its summary. Absent, behaviour is unchanged.\n\n" + + "**Importing from another system (v1.39.2).** Optional `sourceSystem` (`PAPERLESS`, `PAPRA`, `OTHER`) and `sourceId` (its id there, printable, up to 128 characters, never `.` or `..`; needs `sourceSystem`) key the upload, optionally with `sourceInstance` (v1.39.3: the address of the Paperless-ngx or Papra it came from, normalised to its origin; without it the key matches any instance of the system). A re-send with a known key is a duplicate: 200 with the live row (`meta.duplicate: true`), or, when the owner DELETED that document, 200 with `data.deleted: true` and `meta.deleted: true` and nothing stored; this holds after the 30-day purge too. The key may also ride the query string (`?sourceSystem=…&sourceId=…`): a key the vault already holds is then answered at once, the file is not stored or processed, and the request counts only toward the lookup allowance shared with `GET /api/documents/inbound/source` (5000 an hour per token, per person for a session; 429 past it), not toward the hourly upload limit. A key in both places must agree (else 422). Every other upload counts toward the hourly upload limit, duplicates included. A key that is answered with an existing document by its bytes is remembered for that document, so deleting it keeps that key deleted too; one document takes at most 20 such keys, beyond which the upload is refused 409 (`documents.inbound.sourceAliasLimit`). `OTHER` is one shared namespace per account: two different systems both sent as `OTHER` must not reuse each other\'s ids. `aiRead=defer` holds back automatic AI reading for this upload: the thumbnail and a local text index still run, the summary and lab staging do not, and the document stays out of every automatic AI path (the summary catch-up, the automatic index, Index all for search) until the person reads it with AI or generates its summary. Absent, behaviour is unchanged.\n\n" + "**Narrow token.** Also reachable with a `documents:write` Bearer (minted at `POST /api/tokens/documents`), which reaches only this route and the source-key lookup `GET /api/documents/inbound/source`. Such a caller draws on a bucket of its own, keyed on the token (default 120 an hour, `DOCUMENT_UPLOAD_LIMIT_PER_HOUR`, clamped 1-1000; the 429 carries `Retry-After` and `X-RateLimit-*`), and gets a `DocumentUploadReceipt` instead of the stored row; its 413 `quotaExceeded` carries no `quotaBytes` / `usedBytes`. The response is an untagged union (row or receipt) on purpose: a discriminator would need a new field on the row every existing client decodes. A cookie or wildcard caller keeps the 60-an-hour per-user bucket and the full row. With the vault module off the answer is 403 `module.disabled` for every caller.", requestBody: { required: true, @@ -461,6 +470,14 @@ export const inboundDocumentPaths: NonNullable = { description: "Optional. The document's id in `sourceSystem`; requires it. Unique per account across live and deleted documents.", }), + sourceInstance: z + .string() + .max(DOCUMENT_SOURCE_INSTANCE_MAX) + .optional() + .meta({ + description: + "Optional; requires `sourceId`. The instance of `sourceSystem` the id belongs to, as its http(s) address; normalised to the origin (`scheme://host[:port]`). The same id in two instances is two documents. Optional: a key stored without an instance (every upload before v1.39.3) matches any instance of its system, and a key sent without one matches any stored instance.", + }), aiRead: z.enum(["defer"]).optional().meta({ description: "Optional. `defer` holds back automatic AI reading for this upload (local index and thumbnail only).", @@ -527,6 +544,14 @@ export const inboundDocumentPaths: NonNullable = { required: true, schema: { type: "string", maxLength: DOCUMENT_SOURCE_ID_MAX }, }, + { + name: "sourceInstance", + in: "query", + required: false, + schema: { type: "string", maxLength: DOCUMENT_SOURCE_INSTANCE_MAX }, + description: + "The instance of `sourceSystem` the id belongs to, as its http(s) address; normalised to the origin (`scheme://host[:port]`). The same id in two instances is two documents. Optional: a key stored without an instance (every upload before v1.39.3) matches any instance of its system, and a key sent without one matches any stored instance.", + }, ], responses: { "200": { diff --git a/src/lib/openapi/routes/index.ts b/src/lib/openapi/routes/index.ts index c1a5a0a21..377f5fb6c 100644 --- a/src/lib/openapi/routes/index.ts +++ b/src/lib/openapi/routes/index.ts @@ -50,6 +50,7 @@ import { cyclePaths } from "./cycle"; import { dashboardWidgetPaths } from "./dashboard"; import { biomarkerPaths } from "./biomarkers"; import { inboundDocumentPaths } from "./documents"; +import { documentSourcePaths } from "./document-sources"; import { devicePaths } from "./devices"; import { exportPaths } from "./export"; import { familyHistoryPaths } from "./family-history"; @@ -170,6 +171,10 @@ export const openApiPaths: NonNullable = { // discover: a copy this instance can no longer decrypt (appended, spread // order is load-bearing). ...adminBackupPaths, + // v1.39.3 — the document picker: a person's connection to Paperless-ngx or + // Papra, searching it and importing picked documents; cookie-only + // (appended, spread order is load-bearing). + ...documentSourcePaths, // The environmental-context overview. Its own module because nothing else // owns the surface (appended, spread order is load-bearing). // diff --git a/src/lib/query-keys/documents.ts b/src/lib/query-keys/documents.ts index 2a407a2dc..c5c8e8dc9 100644 --- a/src/lib/query-keys/documents.ts +++ b/src/lib/query-keys/documents.ts @@ -81,4 +81,27 @@ export const documentKeys = { * toggle; a flip invalidates the document AI capability probe. */ documentsAutoAiRead: () => ["documents", "auto-ai-read"] as const, + /** + * The document picker (#1038): its own top-level prefix, deliberately NOT + * under `["documents"]`. Every vault write invalidates that prefix, and a + * search or tag read here is a request to the person's Paperless-ngx or + * Papra instance; a filing action in the vault must not re-send them. + */ + documentSources: () => ["documentSources"] as const, + /** Availability + the person's connections (`GET /api/documents/sources`). */ + documentSourcesStatus: () => ["documentSources", "status"] as const, + /** One source's tags for the filter. */ + documentSourceTags: (system: string) => + ["documentSources", "tags", system] as const, + /** One page of one source's search. */ + documentSourceSearch: (system: string, search: DocumentSourceSearchKey) => + ["documentSources", "search", system, search] as const, }; + +/** The facets that make one picker search distinct from another. */ +export interface DocumentSourceSearchKey { + q: string; + tag?: string; + from?: string; + to?: string; +} diff --git a/src/lib/validations/backup.ts b/src/lib/validations/backup.ts index d9aa93bfe..c42b5b566 100644 --- a/src/lib/validations/backup.ts +++ b/src/lib/validations/backup.ts @@ -83,6 +83,7 @@ import { } from "@/lib/validations/emergency-profile"; import { DOCUMENT_SOURCE_ID_MAX, + DOCUMENT_SOURCE_INSTANCE_MAX, DOCUMENT_SOURCE_SYSTEMS, } from "@/lib/validations/inbound-documents"; import { REMINDER_EVENT_SOURCES } from "@/lib/measurement-reminders/satisfy"; @@ -1636,6 +1637,14 @@ const documentBackupSchema = z // restore cannot write a system the detail sheet has no name for. sourceSystem: z.enum(DOCUMENT_SOURCE_SYSTEMS).nullable().optional(), sourceId: z.string().max(DOCUMENT_SOURCE_ID_MAX).nullable().optional(), + // v1.39.3 — the source instance (an origin). Length-bounded here and + // normalised on restore; a value that is not an origin restores as null + // (matches any instance) rather than failing the restore. + sourceInstance: z + .string() + .max(DOCUMENT_SOURCE_INSTANCE_MAX) + .nullable() + .optional(), aiReadDeferred: z.boolean().optional(), summary: z.string().nullable().optional(), createdAt: isoDateTime.optional(), diff --git a/src/lib/validations/document-sources.ts b/src/lib/validations/document-sources.ts new file mode 100644 index 000000000..4034d4c39 --- /dev/null +++ b/src/lib/validations/document-sources.ts @@ -0,0 +1,90 @@ +/** + * Request schemas of the document picker (#1038): saving a connection, + * searching a source, importing one picked document. No `userId` field in any + * of them; the owner is always the authenticated session. + */ +import { z } from "zod/v4"; + +import { DOCUMENT_SOURCE_BASE_URL_MAX } from "@/lib/documents/sources/origins"; +import { + DOCUMENT_PICKER_LINK_KINDS, + DOCUMENT_PICKER_MAX_PAGE, + DOCUMENT_PICKER_QUERY_MAX, + PAPRA_ORG_ID, +} from "@/lib/documents/sources/types"; +import { + documentSourceIdSchema, + INBOUND_DOCUMENT_KINDS, +} from "@/lib/validations/inbound-documents"; + +/** Printable ASCII without spaces: an id or a key, never prose. */ +const printable = /^[\x21-\x7e]+$/u; + +const isoDate = z + .string() + .regex(/^\d{4}-\d{2}-\d{2}$/u, "Expected a YYYY-MM-DD date"); + +/** Longest API token or key a connection stores. */ +export const DOCUMENT_SOURCE_TOKEN_MAX = 512; + +/** + * Save a connection. `token` may be left out when a connection for the system + * already exists: the stored one is kept, so the address or the organization + * can change without typing the key again. The first save must carry it (the + * route says so). `organizationId` is Papra's and required there. + */ +export const documentSourceSaveSchema = z.object({ + baseUrl: z.string().trim().min(1).max(DOCUMENT_SOURCE_BASE_URL_MAX), + organizationId: z + .string() + .trim() + .regex(PAPRA_ORG_ID, "Expected a Papra organization id (org_…)") + .optional(), + token: z + .string() + .trim() + .min(1) + .max(DOCUMENT_SOURCE_TOKEN_MAX) + .regex(printable, "Expected printable characters without spaces") + .optional(), +}); +export type DocumentSourceSaveInput = z.infer; + +export const documentSourceSearchSchema = z + .object({ + q: z.string().trim().max(DOCUMENT_PICKER_QUERY_MAX).default(""), + tag: z + .string() + .trim() + .min(1) + .max(64) + .regex(printable, "Expected printable characters without spaces") + .optional(), + from: isoDate.optional(), + to: isoDate.optional(), + page: z.coerce + .number() + .int() + .min(1) + .max(DOCUMENT_PICKER_MAX_PAGE) + .default(1), + }) + .refine((v) => !v.from || !v.to || v.from <= v.to, { + path: ["to"], + message: "The end date is before the start date", + }); + +export const documentSourceImportSchema = z.object({ + sourceId: documentSourceIdSchema, + kind: z.enum(INBOUND_DOCUMENT_KINDS).optional(), + /** The record the picker was opened from, to link the document to. */ + link: z + .object({ + kind: z.enum(DOCUMENT_PICKER_LINK_KINDS), + id: z.string().trim().min(1).max(64), + }) + .optional(), +}); +export type DocumentSourceImportInput = z.infer< + typeof documentSourceImportSchema +>; diff --git a/src/lib/validations/inbound-documents.ts b/src/lib/validations/inbound-documents.ts index cd78d826a..afe29cd78 100644 --- a/src/lib/validations/inbound-documents.ts +++ b/src/lib/validations/inbound-documents.ts @@ -701,12 +701,46 @@ export const DOCUMENT_SOURCE_ID_MAX = 128; * The document's id in its source system. Printable ASCII only: it is a key, * not prose, and it is echoed on the detail sheet. */ -const documentSourceIdSchema = z +export const documentSourceIdSchema = z .string() .trim() .min(1) .max(DOCUMENT_SOURCE_ID_MAX) - .regex(/^[\x21-\x7e]+$/u, "Expected printable characters without spaces"); + .regex(/^[\x21-\x7e]+$/u, "Expected printable characters without spaces") + // A path segment of its own name: an importer that builds a URL from the id + // must never be handed one that walks up the source's path. + .refine((v) => v !== "." && v !== "..", "Not a document id"); + +/** Longest source instance (an origin) a key may carry. */ +export const DOCUMENT_SOURCE_INSTANCE_MAX = 512; + +/** + * v1.39.3 — the instance of the source system a key belongs to, as the + * normalised origin (`scheme://host[:port]`) of the Paperless-ngx or Papra it + * came from, or null when the value is not one. A path, query or credentials + * in the value are dropped: two addresses on one origin are one instance. + */ +export function normaliseSourceInstance(value: string): string | null { + try { + const url = new URL(value.trim()); + if (url.protocol !== "http:" && url.protocol !== "https:") return null; + if (url.username || url.password) return null; + return url.origin; + } catch { + return null; + } +} + +/** The instance as a request field: validated, then normalised to its origin. */ +export const documentSourceInstanceSchema = z + .string() + .trim() + .min(1) + .max(DOCUMENT_SOURCE_INSTANCE_MAX) + .refine((v) => normaliseSourceInstance(v) !== null, { + message: "Expected the http(s) address of the source instance", + }) + .transform((v) => normaliseSourceInstance(v) as string); /** * A source key on its own — the query-string form the upload accepts ahead of @@ -715,6 +749,8 @@ const documentSourceIdSchema = z export const documentSourceKeySchema = z.object({ sourceSystem: z.enum(DOCUMENT_SOURCE_SYSTEMS), sourceId: documentSourceIdSchema, + /** Optional: without it the key matches any instance of the system. */ + sourceInstance: documentSourceInstanceSchema.optional(), }); export const documentCreateSchema = z @@ -731,6 +767,11 @@ export const documentCreateSchema = z * not prose, and it is echoed on the detail sheet. */ sourceId: documentSourceIdSchema.optional(), + /** + * v1.39.3 — which instance of `sourceSystem` (its origin). Optional; a + * key without one matches any instance. + */ + sourceInstance: documentSourceInstanceSchema.optional(), /** * `defer` holds back automatic AI reading for this upload: the thumbnail * and a local text index still run, the summary and the lab staging do @@ -741,6 +782,10 @@ export const documentCreateSchema = z .refine((v) => v.sourceId === undefined || v.sourceSystem !== undefined, { path: ["sourceSystem"], message: "sourceId needs a sourceSystem", + }) + .refine((v) => v.sourceInstance === undefined || v.sourceId !== undefined, { + path: ["sourceInstance"], + message: "sourceInstance needs a sourceId", }); export type DocumentCreateInput = z.infer; diff --git a/tests/integration/document-sources.test.ts b/tests/integration/document-sources.test.ts new file mode 100644 index 000000000..70776ef58 --- /dev/null +++ b/tests/integration/document-sources.test.ts @@ -0,0 +1,838 @@ +/** + * The document picker (#1038) against a real Postgres, the real auth resolver + * and the real routes. Only the network is replaced: `safeFetch` answers from + * an in-memory Paperless-ngx, so every assertion about what was fetched (and + * what was not) is a count on that fake. + * + * 1. With `DOCUMENT_SOURCE_ORIGINS` unset the picker is off. + * 2. Saving refuses an unlisted origin and a refused token, stores nothing + * then, and otherwise stores the token encrypted and never returns it. + * 3. Only the owner's browser session reaches the routes: a `["*"]` Bearer + * and a delegate acting on the owner's record are refused. + * 4. Search reports what the vault holds per source key; import stores + * through the upload's ingest path with provenance; a re-import is a + * duplicate and a document deleted in HealthLog stays deleted, both + * without a download. + * 5. An import links to the record it was picked for, and a foreign record + * is refused before anything is fetched. + * 6. An origin the operator removed cuts the stored connection off. + */ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { NextRequest } from "next/server"; + +process.env.ENCRYPTION_KEY ??= + "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"; +process.env.API_TOKEN_HMAC_KEY ??= + "test-hmac-key-document-sources-32-bytes-min-0987654321abcd"; + +import { cookieJar, headerJar } from "./mock-next-headers"; +import { getPrismaClient, truncateAllTables } from "./setup"; + +vi.mock("next/headers", async () => { + const { cookieJar, headerJar } = await import("./mock-next-headers"); + return { + headers: vi.fn(async () => ({ + get: (name: string) => headerJar.get(name.toLowerCase()) ?? null, + })), + cookies: vi.fn(async () => ({ + get: (name: string) => { + const value = cookieJar.get(name); + return value ? { name, value } : undefined; + }, + set: (name: string, value: string) => { + cookieJar.set(name, value); + }, + delete: (name: string) => { + cookieJar.delete(name); + }, + })), + }; +}); + +vi.mock("@/lib/db-compat", () => ({ + ensureDbCompatibility: vi.fn().mockResolvedValue(undefined), +})); + +// ── The fake Paperless-ngx behind `safeFetch` ────────────────────────────── + +const ORIGIN = "http://paperless.lan:8000"; +/** A second Paperless-ngx the operator also listed; same ids, other documents. */ +const ORIGIN_B = "https://paperless-b.example.com"; +const TOKEN = "paperless-token-0123456789-secret"; + +const PDF = (tag: string) => + Buffer.from(`%PDF-1.7\n% ${tag}\n1 0 obj\n<<>>\nendobj\n%%EOF\n`); + +const DOCS = [ + { + id: 41, + title: "Blood test", + created: "2024-03-04", + tags: [7], + original_file_name: "blood.pdf", + bytes: PDF("blood"), + }, + { + id: 42, + title: "Discharge letter", + created: "2023-11-20", + tags: [], + original_file_name: "discharge.pdf", + bytes: PDF("discharge"), + }, +]; + +const fake = vi.hoisted(() => ({ + calls: [] as Array<{ url: string; opts: Record }>, + acceptToken: "", + /** Delay a download so two imports of one key overlap. */ + downloadDelayMs: 0, + /** Answer downloads with this status instead of the file. */ + downloadStatus: 200, +})); + +vi.mock("@/lib/safe-fetch", async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + safeFetch: vi.fn( + async ( + target: string, + init: RequestInit, + opts: Record, + ) => { + fake.calls.push({ url: String(target), opts }); + const url = new URL(String(target)); + if (url.origin !== opts.operatorApprovedPrivateOrigin) { + throw new actual.SafeFetchError("not approved", "private_host"); + } + const auth = (init.headers as Record).Authorization; + if (auth !== `Token ${fake.acceptToken}`) { + return new Response("{}", { status: 401 }); + } + const json = (body: unknown) => + new Response(JSON.stringify(body), { + headers: { "content-type": "application/json" }, + }); + const path = url.pathname; + if (path === "/api/tags/") { + return json({ results: [{ id: 7, name: "Health" }], next: null }); + } + if (path === "/api/documents/") { + // Tag 99: a server that ignores the tag filter, with the only match + // on its second page. + if (url.searchParams.get("tags__id__all") === "99") { + const page2 = url.searchParams.get("page") === "2"; + return json({ + next: page2 ? null : "next", + results: page2 + ? [ + { + id: 41, + title: "Blood test", + created: "2024-03-04", + tags: [99], + }, + ] + : [ + { + id: 42, + title: "Discharge letter", + created: "2023-11-20", + tags: [], + }, + ], + }); + } + const q = ( + url.searchParams.get("title__icontains") ?? "" + ).toLowerCase(); + return json({ + next: null, + results: DOCS.filter((d) => d.title.toLowerCase().includes(q)).map( + ({ bytes: _bytes, ...doc }) => doc, + ), + }); + } + const download = /^\/api\/documents\/(\d+)\/download\/$/.exec(path); + if (download) { + const doc = DOCS.find((d) => String(d.id) === download[1]); + if (fake.downloadDelayMs > 0) { + await new Promise((r) => setTimeout(r, fake.downloadDelayMs)); + } + if (fake.downloadStatus !== 200) { + return new Response("", { status: fake.downloadStatus }); + } + // Each instance holds different bytes under the same id. + return doc + ? new Response( + new Uint8Array( + Buffer.concat([doc.bytes, Buffer.from(`% ${url.origin}\n`)]), + ), + ) + : new Response("", { status: 404 }); + } + const detail = /^\/api\/documents\/(\d+)\/$/.exec(path); + if (detail) { + const doc = DOCS.find((d) => String(d.id) === detail[1]); + if (!doc) return new Response("{}", { status: 404 }); + const { bytes: _bytes, ...meta } = doc; + return json(meta); + } + return new Response("{}", { status: 404 }); + }, + ), + }; +}); + +import { decrypt } from "@/lib/crypto"; +import { hashToken } from "@/lib/auth/hmac"; +import { acceptGrant, inviteGrant } from "@/lib/sharing/grants"; + +// ── Helpers ──────────────────────────────────────────────────────────────── + +let userId = ""; +let sessionId = ""; + +async function makeUser(name: string) { + return getPrismaClient().user.create({ + data: { + username: name, + email: `${name}@example.test`, + role: "USER", + timezone: "UTC", + modulePreferencesJson: { inboundDocuments: true }, + }, + }); +} + +async function signIn(id: string) { + return getPrismaClient().session.create({ + data: { userId: id, expiresAt: new Date(Date.now() + 600_000) }, + }); +} + +async function seedOwner() { + const user = await makeUser("owner"); + const session = await signIn(user.id); + userId = user.id; + sessionId = session.id; + asCookie(sessionId); +} + +function asCookie(id: string) { + headerJar.clear(); + cookieJar.set("healthlog_session", id); +} + +type Params = { params: Promise<{ system: string }> }; +const paperless: Params = { + params: Promise.resolve({ system: "paperless" }), +}; +type Handler = (r: NextRequest, c: Params) => Promise; + +function jsonRequest(url: string, method: string, body?: unknown) { + const headers: Record = {}; + const auth = headerJar.get("authorization"); + if (auth) headers.authorization = auth; + if (body !== undefined) headers["content-type"] = "application/json"; + return new NextRequest(`https://health.example${url}`, { + method, + headers, + body: body === undefined ? undefined : JSON.stringify(body), + } as never); +} + +async function status(): Promise { + const { GET } = await import("@/app/api/documents/sources/route"); + return (GET as unknown as (r: NextRequest) => Promise)( + jsonRequest("/api/documents/sources", "GET"), + ); +} + +async function save(body: unknown): Promise { + const { PUT } = await import("@/app/api/documents/sources/[system]/route"); + return (PUT as unknown as Handler)( + jsonRequest("/api/documents/sources/paperless", "PUT", body), + paperless, + ); +} + +async function disconnect(): Promise { + const { DELETE } = await import("@/app/api/documents/sources/[system]/route"); + return (DELETE as unknown as Handler)( + jsonRequest("/api/documents/sources/paperless", "DELETE"), + paperless, + ); +} + +async function search(query = ""): Promise { + const { GET } = + await import("@/app/api/documents/sources/[system]/search/route"); + return (GET as unknown as Handler)( + jsonRequest(`/api/documents/sources/paperless/search?${query}`, "GET"), + paperless, + ); +} + +async function importDoc(body: unknown): Promise { + const { POST } = + await import("@/app/api/documents/sources/[system]/import/route"); + return (POST as unknown as Handler)( + jsonRequest("/api/documents/sources/paperless/import", "POST", body), + paperless, + ); +} + +async function connect() { + fake.acceptToken = TOKEN; + const res = await save({ baseUrl: `${ORIGIN}/`, token: TOKEN }); + expect(res.status).toBe(200); + return res; +} + +function downloads(): number { + return fake.calls.filter((c) => c.url.includes("/download/")).length; +} + +beforeEach(async () => { + await truncateAllTables(getPrismaClient()); + cookieJar.clear(); + headerJar.clear(); + fake.calls.length = 0; + fake.acceptToken = TOKEN; + fake.downloadDelayMs = 0; + fake.downloadStatus = 200; + process.env.DOCUMENT_SOURCE_ORIGINS = `${ORIGIN},${ORIGIN_B}`; +}); + +afterEach(() => { + delete process.env.DOCUMENT_SOURCE_ORIGINS; +}); + +// ── Tests ────────────────────────────────────────────────────────────────── + +describe("the operator's list switches the picker on", () => { + it("is off without DOCUMENT_SOURCE_ORIGINS, and nothing is dialled", async () => { + delete process.env.DOCUMENT_SOURCE_ORIGINS; + await seedOwner(); + const off = await status(); + expect(off.status).toBe(200); + expect((await off.json()).data).toEqual({ + available: false, + allowedOrigins: [], + connections: [], + }); + const refused = await save({ baseUrl: ORIGIN, token: TOKEN }); + expect(refused.status).toBe(404); + expect((await refused.json()).meta.errorCode).toBe( + "documents.sources.unavailable", + ); + expect(fake.calls).toHaveLength(0); + }); +}); + +describe("saving a connection", () => { + it("refuses an origin the operator did not list, public or private, without dialling", async () => { + await seedOwner(); + for (const baseUrl of [ + "http://10.0.0.9:8000", + "https://paperless.example.org", + "http://paperless.lan:8001", + ]) { + const res = await save({ baseUrl, token: TOKEN }); + expect(res.status, baseUrl).toBe(422); + expect((await res.json()).meta.errorCode).toBe( + "documents.sources.originNotAllowed", + ); + } + expect(fake.calls).toHaveLength(0); + expect(await getPrismaClient().documentSourceConnection.count()).toBe(0); + }); + + it("refuses a token the source rejects and stores nothing", async () => { + await seedOwner(); + fake.acceptToken = "the-real-one"; + const res = await save({ baseUrl: ORIGIN, token: TOKEN }); + expect(res.status).toBe(502); + expect((await res.json()).meta).toMatchObject({ + errorCode: "documents.sources.authRefused", + upstreamStatus: 401, + }); + expect(await getPrismaClient().documentSourceConnection.count()).toBe(0); + }); + + it("stores the token encrypted and never returns it", async () => { + await seedOwner(); + const saved = await connect(); + const savedText = await saved.text(); + expect(savedText).not.toContain(TOKEN); + expect(JSON.parse(savedText).data).toMatchObject({ + system: "PAPERLESS", + baseUrl: ORIGIN, + hasToken: true, + originAllowed: true, + }); + + const row = + await getPrismaClient().documentSourceConnection.findFirstOrThrow(); + expect(row.userId).toBe(userId); + expect(row.tokenEncrypted).not.toContain(TOKEN); + expect(decrypt(row.tokenEncrypted)).toBe(TOKEN); + + const read = await (await status()).text(); + expect(read).not.toContain(TOKEN); + expect(JSON.parse(read).data.allowedOrigins).toEqual([ORIGIN, ORIGIN_B]); + + // A save without a token keeps the stored one. + const again = await save({ baseUrl: ORIGIN }); + expect(again.status).toBe(200); + const kept = + await getPrismaClient().documentSourceConnection.findFirstOrThrow(); + expect(decrypt(kept.tokenEncrypted)).toBe(TOKEN); + + expect((await disconnect()).status).toBe(200); + expect(await getPrismaClient().documentSourceConnection.count()).toBe(0); + }); +}); + +describe("only the owner's browser session", () => { + it("refuses a wildcard Bearer token", async () => { + await seedOwner(); + await connect(); + const raw = "hlk_document-sources-wildcard-token-000"; + await getPrismaClient().apiToken.create({ + data: { + userId, + name: "native", + tokenHash: hashToken(raw), + permissions: ["*"], + }, + }); + cookieJar.clear(); + headerJar.set("authorization", `Bearer ${raw}`); + fake.calls.length = 0; + for (const res of [ + await status(), + await search("q=blood"), + await importDoc({ sourceId: "41" }), + ]) { + expect(res.status).toBe(403); + expect((await res.json()).meta.errorCode).toBe( + "documents.sources.browserOnly", + ); + } + expect(fake.calls).toHaveLength(0); + }); + + it("refuses a delegate acting on the owner's record", async () => { + await seedOwner(); + await connect(); + const delegate = await makeUser("delegate"); + const invited = await inviteGrant({ + grantorId: userId, + granteeId: delegate.id, + access: "WRITE", + scope: null, + }); + await acceptGrant({ grantId: invited.id, granteeId: delegate.id }); + const session = await signIn(delegate.id); + const switched = await getPrismaClient().session.update({ + where: { id: session.id }, + data: { actingAsUserId: userId }, + }); + asCookie(session.id); + headerJar.set("x-healthlog-record-epoch", String(switched.recordEpoch)); + headerJar.set("x-healthlog-record-scope", userId); + fake.calls.length = 0; + + for (const res of [ + await status(), + await search("q=blood"), + await importDoc({ sourceId: "41" }), + ]) { + expect(res.status).toBe(403); + } + expect(fake.calls).toHaveLength(0); + expect(await getPrismaClient().inboundDocument.count()).toBe(0); + }); +}); + +describe("search and import", () => { + it("imports through the ingest path; a re-import is a duplicate and a deleted one stays deleted", async () => { + await seedOwner(); + await connect(); + const prisma = getPrismaClient(); + + const first = await search("q=blood"); + expect(first.status).toBe(200); + expect((await first.json()).data).toEqual({ + page: 1, + hasMore: false, + results: [ + { + sourceId: "41", + title: "Blood test", + date: "2024-03-04", + tags: ["Health"], + sizeBytes: null, + state: "new", + documentId: null, + }, + ], + }); + + const imported = await importDoc({ sourceId: "41" }); + expect(imported.status).toBe(201); + const receipt = (await imported.json()).data; + expect(receipt).toMatchObject({ outcome: "imported", linked: false }); + const stored = await prisma.inboundDocument.findUniqueOrThrow({ + where: { id: receipt.documentId }, + omit: { contentEncrypted: true }, + }); + expect(stored).toMatchObject({ + userId, + title: "Blood test", + filename: "blood.pdf", + sourceSystem: "PAPERLESS", + sourceId: "41", + mimeType: "application/pdf", + aiReadDeferred: false, + kind: "OTHER", + }); + expect(stored.documentDate?.toISOString().slice(0, 10)).toBe("2024-03-04"); + expect(downloads()).toBe(1); + + const listed = (await (await search("q=blood")).json()).data.results[0]; + expect(listed).toMatchObject({ + state: "imported", + documentId: receipt.documentId, + }); + + const again = await importDoc({ sourceId: "41" }); + expect(again.status).toBe(200); + expect((await again.json()).data).toEqual({ + outcome: "duplicate", + documentId: receipt.documentId, + linked: false, + }); + expect(downloads()).toBe(1); + + await prisma.inboundDocument.update({ + where: { id: receipt.documentId }, + data: { deletedAt: new Date() }, + }); + expect( + (await (await search("q=blood")).json()).data.results[0], + ).toMatchObject({ state: "deleted", documentId: null }); + const revived = await importDoc({ sourceId: "41" }); + expect(revived.status).toBe(200); + expect((await revived.json()).data.outcome).toBe("deleted"); + expect(downloads()).toBe(1); + expect(await prisma.inboundDocument.count()).toBe(1); + }); + + it("answers a file already uploaded by hand as a duplicate and remembers the key", async () => { + await seedOwner(); + await connect(); + const prisma = getPrismaClient(); + + // The same bytes, uploaded through the vault without a source key. + const form = new FormData(); + form.append( + "file", + new Blob([ + new Uint8Array( + Buffer.concat([DOCS[1].bytes, Buffer.from(`% ${ORIGIN}\n`)]), + ), + ]), + "x.pdf", + ); + const { POST } = await import("@/app/api/documents/inbound/route"); + const uploaded = await ( + POST as unknown as (r: Request) => Promise + )( + new Request("http://localhost/api/documents/inbound", { + method: "POST", + body: form, + }), + ); + expect(uploaded.status).toBe(201); + const uploadedId = (await uploaded.json()).data.id; + + const picked = await importDoc({ sourceId: "42" }); + expect(picked.status).toBe(200); + expect((await picked.json()).data).toMatchObject({ + outcome: "duplicate", + documentId: uploadedId, + }); + expect(await prisma.inboundDocument.count()).toBe(1); + expect( + await prisma.documentSourceAlias.count({ + where: { + documentId: uploadedId, + sourceSystem: "PAPERLESS", + sourceInstance: ORIGIN, + sourceId: "42", + }, + }), + ).toBe(1); + }); + + it("links to the visit it was picked for, and refuses someone else's before fetching", async () => { + await seedOwner(); + await connect(); + const prisma = getPrismaClient(); + const visit = await prisma.encounter.create({ + data: { userId, occurredAt: new Date("2024-03-04T10:00:00Z") }, + }); + const stranger = await makeUser("stranger"); + const foreign = await prisma.encounter.create({ + data: { userId: stranger.id, occurredAt: new Date() }, + }); + + const refused = await importDoc({ + sourceId: "41", + link: { kind: "encounter", id: foreign.id }, + }); + expect(refused.status).toBe(404); + expect((await refused.json()).meta.errorCode).toBe( + "documents.sources.linkTargetNotFound", + ); + expect(downloads()).toBe(0); + + const linked = await importDoc({ + sourceId: "41", + link: { kind: "encounter", id: visit.id }, + }); + expect(linked.status).toBe(201); + const data = (await linked.json()).data; + expect(data.linked).toBe(true); + expect( + await prisma.encounterDocumentLink.count({ + where: { encounterId: visit.id, documentId: data.documentId }, + }), + ).toBe(1); + }); + + it("cuts a stored connection off once the operator removes its origin", async () => { + await seedOwner(); + await connect(); + fake.calls.length = 0; + process.env.DOCUMENT_SOURCE_ORIGINS = "https://other.example.com"; + + const res = await search("q=blood"); + expect(res.status).toBe(422); + expect((await res.json()).meta.errorCode).toBe( + "documents.sources.originNotAllowed", + ); + const imported = await importDoc({ sourceId: "41" }); + expect(imported.status).toBe(422); + expect(fake.calls).toHaveLength(0); + + const read = (await (await status()).json()).data; + expect(read.connections[0]).toMatchObject({ originAllowed: false }); + // Still removable. + expect((await disconnect()).status).toBe(200); + }); +}); + +describe("review follow-ups", () => { + it("keeps a document deleted in HealthLog deleted when the same bytes arrive under another source key", async () => { + await seedOwner(); + await connect(); + const prisma = getPrismaClient(); + const first = await importDoc({ sourceId: "41" }); + const id = (await first.json()).data.documentId as string; + await prisma.inboundDocument.update({ + where: { id }, + data: { deletedAt: new Date() }, + }); + + // The same file, uploaded by another importer under its own key (a + // document token or the script sending from another system). + const form = new FormData(); + form.append( + "file", + new Blob([ + new Uint8Array( + Buffer.concat([DOCS[0].bytes, Buffer.from(`% ${ORIGIN}\n`)]), + ), + ]), + "x.pdf", + ); + form.append("sourceSystem", "PAPRA"); + form.append("sourceId", "doc_same_bytes"); + const { POST } = await import("@/app/api/documents/inbound/route"); + const res = await (POST as unknown as (r: Request) => Promise)( + new Request("http://localhost/api/documents/inbound", { + method: "POST", + body: form, + }), + ); + expect(res.status).toBe(200); + expect((await res.json()).data).toMatchObject({ deleted: true }); + expect(await prisma.inboundDocument.count()).toBe(1); + // The new key is remembered on the deleted document, so the purge + // carries it into the ledger. + expect( + await prisma.documentSourceAlias.count({ + where: { documentId: id, sourceId: "doc_same_bytes" }, + }), + ).toBe(1); + + // A person uploading the same file by hand, without a key, is a + // deliberate new copy and is stored. + const manual = new FormData(); + manual.append( + "file", + new Blob([ + new Uint8Array( + Buffer.concat([DOCS[0].bytes, Buffer.from(`% ${ORIGIN}\n`)]), + ), + ]), + "x.pdf", + ); + const byHand = await (POST as unknown as (r: Request) => Promise)( + new Request("http://localhost/api/documents/inbound", { + method: "POST", + body: manual, + }), + ); + expect(byHand.status).toBe(201); + }); + + it("never sends the stored token to a new origin", async () => { + await seedOwner(); + await connect(); + fake.calls.length = 0; + const moved = await save({ baseUrl: ORIGIN_B }); + expect(moved.status).toBe(422); + expect((await moved.json()).meta.errorCode).toBe( + "documents.sources.tokenRequired", + ); + expect(fake.calls).toHaveLength(0); + // A new path on the same origin keeps the token. + const samePlace = await save({ baseUrl: `${ORIGIN}/` }); + expect(samePlace.status).toBe(200); + expect(fake.calls.every((c) => c.url.startsWith(ORIGIN))).toBe(true); + }); + + it("keeps the same id on two instances apart, and lets a key without an instance match any", async () => { + await seedOwner(); + await connect(); + const prisma = getPrismaClient(); + + const first = await importDoc({ sourceId: "41" }); + expect(first.status).toBe(201); + const a = (await first.json()).data.documentId as string; + expect( + (await prisma.inboundDocument.findUniqueOrThrow({ where: { id: a } })) + .sourceInstance, + ).toBe(ORIGIN); + + // Switch the connection to the other Paperless-ngx. + expect((await save({ baseUrl: ORIGIN_B, token: TOKEN })).status).toBe(200); + const listed = (await (await search("q=blood")).json()).data.results[0]; + expect(listed).toMatchObject({ sourceId: "41", state: "new" }); + const second = await importDoc({ sourceId: "41" }); + expect(second.status).toBe(201); + const b = (await second.json()).data.documentId as string; + expect(b).not.toBe(a); + expect( + (await prisma.inboundDocument.findUniqueOrThrow({ where: { id: b } })) + .sourceInstance, + ).toBe(ORIGIN_B); + + // A v1.39.2 import of #42 carries no instance: it answers for any + // instance of its system, so the picker neither re-imports it nor + // forgets a deletion of it. + await prisma.inboundDocument.create({ + data: { + userId, + kind: "OTHER", + mimeType: "application/pdf", + byteSize: 1, + contentEncrypted: Buffer.from("x"), + contentSha256: "legacy-sha", + status: "STORED", + sourceSystem: "PAPERLESS", + sourceId: "42", + deletedAt: new Date(), + }, + }); + const legacy = (await (await search("q=discharge")).json()).data.results[0]; + expect(legacy).toMatchObject({ sourceId: "42", state: "deleted" }); + const refused = await importDoc({ sourceId: "42" }); + expect((await refused.json()).data.outcome).toBe("deleted"); + }); + + it("refuses an id that could leave the document path, before dialling", async () => { + await seedOwner(); + await connect(); + fake.calls.length = 0; + for (const sourceId of ["..", ".", "41/../../admin", "abc", "41?x=1"]) { + const res = await importDoc({ sourceId }); + expect(res.status, sourceId).toBe(422); + } + expect(fake.calls).toHaveLength(0); + }); + + it("downloads a key once when two imports of it overlap", async () => { + await seedOwner(); + await connect(); + fake.downloadDelayMs = 300; + const [one, two] = await Promise.all([ + importDoc({ sourceId: "41" }), + importDoc({ sourceId: "41" }), + ]); + const outcomes = [ + (await one.json()).data.outcome, + (await two.json()).data.outcome, + ].sort(); + expect(outcomes).toEqual(["duplicate", "imported"]); + expect(downloads()).toBe(1); + expect(await getPrismaClient().inboundDocument.count()).toBe(1); + }); + + it("does not charge the upload allowance for a fetch that failed", async () => { + await seedOwner(); + await connect(); + fake.downloadStatus = 500; + const res = await importDoc({ sourceId: "41" }); + expect(res.status).toBe(502); + expect( + await getPrismaClient().rateLimit.count({ + where: { key: `documents-upload:${userId}` }, + }), + ).toBe(0); + }); + + it("reads past a page the local re-check emptied", async () => { + await seedOwner(); + await connect(); + const res = await search("tag=99"); + expect(res.status).toBe(200); + const data = (await res.json()).data; + expect(data.page).toBe(2); + expect(data.results.map((r: { sourceId: string }) => r.sourceId)).toEqual([ + "41", + ]); + }); + + it("shows and deletes a connection with the module off and the list gone", async () => { + await seedOwner(); + await connect(); + delete process.env.DOCUMENT_SOURCE_ORIGINS; + await getPrismaClient().user.update({ + where: { id: userId }, + data: { modulePreferencesJson: { inboundDocuments: false } }, + }); + const read = (await (await status()).json()).data; + expect(read.available).toBe(false); + expect(read.connections).toHaveLength(1); + expect(read.connections[0].originAllowed).toBe(false); + expect((await disconnect()).status).toBe(200); + expect(await getPrismaClient().documentSourceConnection.count()).toBe(0); + }); +});