diff --git a/.env.example b/.env.example index aa5a7e3c3..5474388c2 100644 --- a/.env.example +++ b/.env.example @@ -103,6 +103,20 @@ API_TOKEN_HMAC_KEY="REPLACE-WITH-OUTPUT-OF-openssl-rand-hex-32" # NOTIFICATION_PRIVATE_ORIGINS="" +# ----------------------------------------------------------------------------- +# Document picker: Paperless-ngx and Papra -- optional +# ----------------------------------------------------------------------------- +# The Paperless-ngx and Papra instances people may connect to pick documents +# from, comma-separated exact origins (scheme://host[:port], no path), e.g. +# "https://paperless.example.com,http://papra.lan:1221". Unset, the document +# picker is switched off. Only a listed origin is ever contacted, public or +# private; requests are pinned to it, redirects are not followed, and +# link-local, metadata and unspecified addresses stay refused even when +# listed. A malformed entry is logged once and grants nothing. +# See docs/self-hosting/document-sources.md. +# DOCUMENT_SOURCE_ORIGINS="" + + # ----------------------------------------------------------------------------- # AI Provider (optional) # ----------------------------------------------------------------------------- diff --git a/.env.production.example b/.env.production.example index ead149065..a297815e3 100644 --- a/.env.production.example +++ b/.env.production.example @@ -85,6 +85,20 @@ API_TOKEN_HMAC_KEY="" # NOTIFICATION_PRIVATE_ORIGINS="" +# ----------------------------------------------------------------------------- +# Document picker: Paperless-ngx and Papra -- optional +# ----------------------------------------------------------------------------- +# The Paperless-ngx and Papra instances people may connect to pick documents +# from, comma-separated exact origins (scheme://host[:port], no path), e.g. +# "https://paperless.example.com,http://papra.lan:1221". Unset, the document +# picker is switched off. Only a listed origin is ever contacted, public or +# private; requests are pinned to it, redirects are not followed, and +# link-local, metadata and unspecified addresses stay refused even when +# listed. A malformed entry is logged once and grants nothing. +# See docs/self-hosting/document-sources.md. +# DOCUMENT_SOURCE_ORIGINS="" + + # ----------------------------------------------------------------------------- # WHOOP -- optional integration # ----------------------------------------------------------------------------- diff --git a/docker-compose.yml b/docker-compose.yml index a24b90ba0..c595de613 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -215,6 +215,13 @@ x-healthlog-env: &healthlog_env # and unspecified addresses stay refused even when listed. See # docs/self-hosting/notifications.md. NOTIFICATION_PRIVATE_ORIGINS: "${NOTIFICATION_PRIVATE_ORIGINS:-}" + # Paperless-ngx and Papra instances the document picker may reach, + # comma-separated exact origins (e.g. "https://paperless.example.com, + # http://papra.lan:1221"). Unset, the picker is off. Only listed origins are + # ever contacted, public or private; redirects are not followed and + # link-local, metadata and unspecified addresses stay refused even when + # listed. See docs/self-hosting/document-sources.md. + DOCUMENT_SOURCE_ORIGINS: "${DOCUMENT_SOURCE_ORIGINS:-}" # Open-Meteo endpoints for the environment/weather correlation. Default to # the hosted service; point them at your own Open-Meteo deployment to keep # the coordinate lookups off a third party. diff --git a/docs/self-hosting/document-sources.md b/docs/self-hosting/document-sources.md new file mode 100644 index 000000000..96588771b --- /dev/null +++ b/docs/self-hosting/document-sources.md @@ -0,0 +1,82 @@ +# Document picker — Paperless-ngx and Papra + +People who keep their paperwork in Paperless-ngx or Papra can search it from +HealthLog and import the documents they pick: under "Link a document" on a +visit, a vaccination or a condition, and from the Documents page. They search +by document name, narrow by tag and date, tick what they want, and HealthLog +fetches those files and stores its own encrypted copy, exactly as if they had +been uploaded. + +The picker is off until the operator names the instances HealthLog may +contact. + +## Turning it on + +List the origins in the server environment: + +```env +DOCUMENT_SOURCE_ORIGINS="https://paperless.example.com,http://papra.lan:1221" +``` + +The variable is on the `docker-compose.yml` whitelist, so setting it in `.env` +is enough; restart the app container afterwards. Unset or empty, the settings +card and every import button stay hidden and the routes answer 404. + +Each entry is one `scheme://host[:port]`, the grammar +`NOTIFICATION_PRIVATE_ORIGINS` uses (see [notifications](notifications.md)): + +- only `http` and `https`; scheme, host and port must match what a person + enters (`https://paperless.example.com:8443` and + `https://paperless.example.com` are two different entries); +- no path, query, credentials, wildcard or CIDR; a person may still enter a + base address with a path (`https://example.com/paperless`) as long as its + origin is listed; +- the unspecified address, link-local and the cloud-metadata range can never + be listed, and a listed name that resolves there is refused when HealthLog + connects; +- a malformed entry is logged once at startup, reduced to scheme and host, and + grants nothing; the valid entries beside it keep working. + +Unlike the notification list, this list is the whole decision: an origin +that is not listed is refused even when it is a public address. Removing an +entry cuts off the connections that point at it on their next request; people +can still delete their stored connection. + +## What each person sets up + +Settings → Integrations → Document archives: + +- **Paperless-ngx:** the base address and an API token (Paperless-ngx: My + Profile → API token). Paperless-ngx 2.16 or later (API version 9). +- **Papra:** the base address, the organization id (from the Papra address + bar) and an API key with `documents:read` and `tags:read`. + +HealthLog tests the connection before saving it. The token is stored +encrypted (AES-256-GCM, rotated with the other keys by +`scripts/rotate-encryption-key.ts`) and is never shown again, not even in +part. + +## What leaves the server, and when + +Only requests the person starts: a search, the tag list for the filter, a +connection test, and one download per picked document. There is no background +sync and no polling. Every request goes to the listed origin only, with +redirects refused, a 10-second limit for searches and 60 seconds for a file, +and a size cap on every answer (1 MiB for a search page, the vault's per-file +limit for a document). + +## Limits and rules + +- Imports count against the person's normal upload allowance (60 an hour) and + storage quota. +- A document that is already in HealthLog is not stored twice, and one the + person deleted in HealthLog is not brought back by picking it again. These + are the same rules the document token and `scripts/import-documents.mjs` + follow. +- A picked document follows the person's own automatic AI reading setting. +- Only the owner of a record can connect an archive or import from it. People + with shared access to a record, whatever their level, do not see the picker. +- Connections are not part of backups. After a restore on another server, + each person reconnects (one form); the imported documents themselves are in + the backup with the rest of the vault. +- Delete all data and account deletion remove the connection and its token. diff --git a/prisma/migrations/0356_document_source_connections/migration.sql b/prisma/migrations/0356_document_source_connections/migration.sql new file mode 100644 index 000000000..6423236ed --- /dev/null +++ b/prisma/migrations/0356_document_source_connections/migration.sql @@ -0,0 +1,22 @@ +-- Document picker (#1038). One connection per person per system (Paperless-ngx +-- or Papra) with the API token encrypted at rest. The picker searches the +-- source and imports picked documents on request only; nothing polls. +CREATE TABLE "document_source_connections" ( + "id" TEXT NOT NULL, + "user_id" TEXT NOT NULL, + "system" VARCHAR(16) NOT NULL, + "base_url" VARCHAR(2048) NOT NULL, + "organization_id" VARCHAR(128), + "token_encrypted" TEXT NOT NULL, + "last_verified_at" TIMESTAMP(3), + "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" TIMESTAMP(3) NOT NULL, + CONSTRAINT "document_source_connections_pkey" PRIMARY KEY ("id") +); + +CREATE UNIQUE INDEX "document_source_connections_user_id_system_key" + ON "document_source_connections" ("user_id", "system"); + +ALTER TABLE "document_source_connections" + ADD CONSTRAINT "document_source_connections_user_id_fkey" + FOREIGN KEY ("user_id") REFERENCES "users"("id") ON DELETE CASCADE ON UPDATE CASCADE; diff --git a/prisma/schema.prisma b/prisma/schema.prisma index 7d725e8d3..47037b890 100644 --- a/prisma/schema.prisma +++ b/prisma/schema.prisma @@ -928,6 +928,9 @@ model User { // v1.39.2 — further source keys an import answered with an existing // document (same bytes under another key). See DocumentSourceAlias. documentSourceAliases DocumentSourceAlias[] + // v1.39.3 — the person's own connection to a Paperless-ngx or Papra + // instance for the document picker. See DocumentSourceConnection. + documentSourceConnections DocumentSourceConnection[] extractedFacts ExtractedFact[] documentConditionLinks DocumentConditionLink[] documentContentIndexes DocumentContentIndex[] @@ -8114,6 +8117,43 @@ model DocumentImportKey { @@map("document_import_keys") } +/// v1.39.3 (#1038) — one person's connection to a Paperless-ngx or Papra +/// instance, used by the document picker to search that system by name, tag +/// and date and import the documents the person picks. One per system per +/// person. Only reachable when the operator lists the instance's origin in +/// `DOCUMENT_SOURCE_ORIGINS`; the origin is re-checked on every request, so +/// removing it from the list cuts the connection off without touching the +/// row. No background job reads it: every request to the source is one the +/// person started. Not carried in backups (a third-party credential for a +/// service on the operator's network; see `NOT_IN_BACKUP_MODELS`); removed by +/// Delete all data and with the account. +model DocumentSourceConnection { + id String @id @default(cuid()) + userId String @map("user_id") + /// `PAPERLESS` | `PAPRA` (`DOCUMENT_PICKER_SYSTEMS` in + /// `src/lib/documents/sources/types.ts`). + system String @map("system") @db.VarChar(16) + /// Base address of the instance, origin plus an optional path. Plaintext: + /// its origin is compared with the operator's list on every use, and the + /// settings card shows it. Not health data. + baseUrl String @map("base_url") @db.VarChar(2048) + /// Papra organization id; null for Paperless-ngx. + organizationId String? @map("organization_id") @db.VarChar(128) + /// The API token (Paperless-ngx) or API key (Papra), AES-256-GCM via + /// `encrypt()`. No route returns it, in whole or in part. + tokenEncrypted String @map("token_encrypted") + /// Last time the source answered a test with this token: the save's live + /// check or the settings card's Test button. Shown on the card. + lastVerifiedAt DateTime? @map("last_verified_at") + createdAt DateTime @default(now()) @map("created_at") + updatedAt DateTime @updatedAt @map("updated_at") + + user User @relation(fields: [userId], references: [id], onDelete: Cascade) + + @@unique([userId, system]) + @@map("document_source_connections") +} + model ExtractedFact { id String @id @default(cuid()) documentId String @map("document_id") diff --git a/scripts/env-manifest.json b/scripts/env-manifest.json index 11fcfadb5..8992f4abc 100644 --- a/scripts/env-manifest.json +++ b/scripts/env-manifest.json @@ -245,6 +245,17 @@ } ] }, + { + "name": "Document picker", + "description": "Operator grant for the document picker, which lets a person search their own Paperless-ngx or Papra instance and import picked documents. Unset, the picker is off. Optional.", + "required": false, + "variables": [ + { + "name": "DOCUMENT_SOURCE_ORIGINS", + "purpose": "Comma-separated exact origins (scheme://host[:port], no path, no wildcard, no CIDR), e.g. \"https://paperless.example.com,http://papra.lan:1221\". Only listed origins are contacted, public or private; redirects are not followed; link-local, metadata and unspecified addresses stay refused even when listed; a malformed entry is logged once and grants nothing." + } + ] + }, { "name": "Off-host backups", "description": "Required ALL-OR-NONE: any of these without the rest disables backups silently.", diff --git a/scripts/rotate-encryption-key.ts b/scripts/rotate-encryption-key.ts index 0bac08bc9..20fa5124b 100644 --- a/scripts/rotate-encryption-key.ts +++ b/scripts/rotate-encryption-key.ts @@ -328,6 +328,16 @@ async function main() { ); } + // ───── Document picker (String) ───── + // Column: "tokenEncrypted" — a person's Paperless-ngx / Papra API token. + results.push( + await rotateStringColumn( + "DocumentSourceConnection", + "tokenEncrypted", + prisma.documentSourceConnection, + ), + ); + // ───── AppSettings — operator credentials (String) ───── // Columns: "adminAiKeyEncrypted" "webPushVapidPrivateKeyEncrypted" // "adminCodexAccessTokenEncrypted" "adminCodexRefreshTokenEncrypted" diff --git a/src/app/api/documents/sources/[system]/import/route.ts b/src/app/api/documents/sources/[system]/import/route.ts new file mode 100644 index 000000000..a2cad851e --- /dev/null +++ b/src/app/api/documents/sources/[system]/import/route.ts @@ -0,0 +1,294 @@ +/** + * Document picker (#1038): import one picked document from a connected + * archive into the vault. + * + * One document per request, so the sheet can show each one's progress and + * result, and no single request holds more than one file in memory. The path: + * + * 1. the link target, when the picker was opened from a record, must be one + * of the person's own live records (404 before anything leaves the + * server); + * 2. the source key answers first, without a download: a document already + * imported (or already holding this key as an alias) is a `duplicate`, + * and one the person deleted here is `deleted` — nothing is fetched and + * nothing is stored, exactly as the token upload answers the same key; + * 3. only then the vault's own upload allowance (60 an hour, shared with + * uploads from the web and the phone) and an upload slot are taken, the + * document's details and its original file are fetched (bounded by the + * vault's per-file cap), and the file goes through `ingestDocument`, the + * same function the upload route stores through: type check, content + * dedup, quota, provenance (`sourceSystem` / `sourceId`), the background + * jobs; + * 4. the document is linked to the record through `src/lib/links/`. + * + * AI reading follows the person's own setting, unlike the token upload's + * default of holding it back: the person picked this document by hand, as + * deliberately as an upload. The hourly allowance and the daily AI budget + * bound it either way. + * + * Cookie-only and owner-only; see `admitDocumentSourceCaller`. + */ +import { Buffer } from "node:buffer"; + +import { NextRequest, NextResponse } from "next/server"; + +import { apiHandler, requireAuth } from "@/lib/api-handler"; +import { + apiError, + apiValidationError, + getClientIp, + safeJson, + sanitiseZodIssues, +} from "@/lib/api-response"; +import { prisma } from "@/lib/db"; +import { + ingestDocument, + personalUploadBucket, + UPLOAD_LIMIT_PER_HOUR, + UPLOAD_WINDOW_MS, +} from "@/lib/documents/ingest"; +import { findSourceKey, type SourceKeyMatch } from "@/lib/documents/source-key"; +import { clientFor, loadConnection } from "@/lib/documents/sources/connections"; +import { + admitDocumentSourceCaller, + responseForSourceFailure, + sourceErrorResponse, + systemParam, +} from "@/lib/documents/sources/route-support"; +import type { + DocumentImportOutcome, + DocumentPickerLinkKind, + DocumentSourceImportDto, +} from "@/lib/documents/sources/types"; +import { + acquireDocumentUploadSlot, + resolveDocumentLimits, +} from "@/lib/documents/upload-policy"; +import { linkTargets } from "@/lib/links"; +import { annotate } from "@/lib/logging/context"; +import { checkRateLimit, rateLimitHeaders } from "@/lib/rate-limit"; +import { documentSourceImportSchema } from "@/lib/validations/document-sources"; + +export const dynamic = "force-dynamic"; + +type RouteParams = { params: Promise<{ system: string }> }; + +interface LinkTarget { + kind: DocumentPickerLinkKind; + id: string; +} + +/** Is the record the picker was opened from one of the person's live ones? */ +async function ownsLinkTarget( + userId: string, + link: LinkTarget, +): Promise { + const where = { id: link.id, userId, deletedAt: null }; + const select = { id: true } as const; + switch (link.kind) { + case "conditionEpisode": + return ( + (await prisma.illnessEpisode.findFirst({ where, select })) !== null + ); + case "encounter": + return (await prisma.encounter.findFirst({ where, select })) !== null; + case "vaccination": + return ( + (await prisma.vaccinationRecord.findFirst({ where, select })) !== null + ); + } +} + +/** File the document against the record, from the record's side. */ +async function linkDocument( + userId: string, + link: LinkTarget | undefined, + documentId: string, +): Promise { + if (!link) return false; + const result = await prisma.$transaction((tx) => + linkTargets(tx, { + userId, + sourceKind: link.kind, + sourceId: link.id, + targetKind: "document", + targetIds: [documentId], + }), + ); + return !result.unknownSource && result.unknownTargetIds.length === 0; +} + +function respond( + system: string, + outcome: DocumentImportOutcome, + documentId: string | null, + linked: boolean, +): NextResponse { + annotate({ + action: { name: "documents.sources.import" }, + meta: { system, outcome, linked, documentId }, + }); + const data: DocumentSourceImportDto = { outcome, documentId, linked }; + return NextResponse.json( + { data, error: null }, + { status: outcome === "imported" ? 201 : 200 }, + ); +} + +/** A key the vault already holds: link a live one, never revive a deleted one. */ +async function answerHeldKey( + system: string, + userId: string, + link: LinkTarget | undefined, + match: SourceKeyMatch, +): Promise { + if (match.state === "deleted") { + return respond(system, "deleted", match.id, false); + } + const linked = await linkDocument(userId, link, match.document.id); + return respond(system, "duplicate", match.document.id, linked); +} + +export const POST = apiHandler( + async (request: NextRequest, { params }: RouteParams) => { + const auth = await requireAuth(); + const refused = await admitDocumentSourceCaller(auth); + if (refused) return refused; + const system = await systemParam(params); + if (system instanceof Response) return system; + const userId = auth.user.id; + + const { data: body, error: jsonError } = await safeJson(request, { + maxBytes: 4 * 1024, + }); + if (jsonError) return jsonError; + const parsed = documentSourceImportSchema.safeParse(body); + if (!parsed.success) { + return apiValidationError( + "Invalid import request", + sanitiseZodIssues(parsed.error.issues), + 422, + ); + } + const { sourceId, kind, link } = parsed.data; + + const row = await loadConnection(userId, system); + if (!row) return sourceErrorResponse("notConnected"); + let client; + try { + client = clientFor(row); + } catch (err) { + return responseForSourceFailure(err); + } + + if (link && !(await ownsLinkTarget(userId, link))) { + return sourceErrorResponse("linkTargetNotFound"); + } + + // The source key answers before anything is downloaded or charged. + const held = await findSourceKey(userId, system, sourceId); + if (held) return answerHeldKey(system, userId, link, held); + + const rl = await checkRateLimit( + personalUploadBucket(userId), + UPLOAD_LIMIT_PER_HOUR, + UPLOAD_WINDOW_MS, + ); + if (!rl.allowed) { + return apiError("Too many uploads. Try again later.", 429, { + errorCode: "documents.inbound.rateLimited", + headers: rateLimitHeaders(rl), + }); + } + + const release = acquireDocumentUploadSlot(userId); + if (!release) { + return apiError( + "Too many uploads are already in progress. Try again shortly.", + 429, + { + errorCode: "documents.inbound.uploadBusy", + reason: "uploadBusy", + headers: { "Retry-After": "1" }, + }, + ); + } + + try { + const limits = await resolveDocumentLimits(userId); + let meta; + let file; + try { + meta = await client.document(sourceId); + file = await client.download(sourceId, limits.maxFileBytes); + } catch (err) { + return responseForSourceFailure(err); + } + + const result = await ingestDocument({ + userId, + scoped: false, + ipAddress: getClientIp(request), + bytes: Buffer.from(file.bytes), + filename: meta.filename ?? file.filename, + title: meta.title, + kind: kind ?? null, + documentDate: meta.date, + episodeIds: [], + encounterIds: [], + sourceSystem: system, + sourceId, + aiDeferred: false, + sourceKeyChecked: true, + limits, + }); + + switch (result.kind) { + case "sourceKey": + return answerHeldKey(system, userId, link, result.match); + case "duplicate": { + // The same bytes were already in the vault (an upload, or another + // archive); the key is now remembered for that document. + const linked = await linkDocument(userId, link, result.document.id); + return respond(system, "duplicate", result.document.id, linked); + } + case "stored": { + const linked = await linkDocument(userId, link, result.document.id); + return respond(system, "imported", result.document.id, linked); + } + case "tooLarge": + return apiError("File is too large.", 413, { + errorCode: "documents.inbound.fileTooLarge", + reason: "fileTooLarge", + maxFileBytes: result.maxFileBytes, + }); + case "empty": + case "unsupportedType": + return apiError("This file type is not supported.", 415, { + errorCode: "documents.inbound.fileType", + reason: "unsupportedType", + }); + case "quotaExceeded": + return apiError("Storage quota exceeded.", 413, { + errorCode: "documents.inbound.quotaExceeded", + reason: "quotaExceeded", + quotaBytes: result.quotaBytes, + usedBytes: result.usedBytes, + }); + case "aliasLimit": + return apiError( + "This file is already stored under too many source ids.", + 409, + { errorCode: "documents.inbound.sourceAliasLimit" }, + ); + case "episodeNotFound": + case "encounterNotFound": + // No pre-links are passed on this path; unreachable, but a result + // the switch does not name must not fall through silently. + return sourceErrorResponse("linkTargetNotFound"); + } + } finally { + release(); + } + }, +); diff --git a/src/app/api/documents/sources/[system]/route.ts b/src/app/api/documents/sources/[system]/route.ts new file mode 100644 index 000000000..0057e4c24 --- /dev/null +++ b/src/app/api/documents/sources/[system]/route.ts @@ -0,0 +1,191 @@ +/** + * Document picker (#1038): save or remove the person's connection to one + * document archive (`paperless` or `papra`). + * + * PUT validates the base address against the operator's list + * (`DOCUMENT_SOURCE_ORIGINS`), tests the connection live before storing + * anything (a wrong token or an unreachable host is reported here, not on the + * first search), then stores the token encrypted. A PUT without a token keeps + * the stored one, so the address or the Papra organization can change without + * typing the key again; the first save must carry it. The response is the + * connection without its token, like every read of it. + * + * DELETE removes the row, and with it the token. + * + * Cookie-only and owner-only; see `admitDocumentSourceCaller`. + */ +import { NextRequest } from "next/server"; + +import { apiHandler, requireAuth } from "@/lib/api-handler"; +import { + apiError, + apiSuccess, + apiValidationError, + getClientIp, + safeJson, + sanitiseZodIssues, +} from "@/lib/api-response"; +import { auditLog } from "@/lib/auth/audit"; +import { decrypt, encrypt } from "@/lib/crypto"; +import { prisma } from "@/lib/db"; +import { sourceClient } from "@/lib/documents/sources/clients"; +import { + loadConnection, + toConnectionDto, +} from "@/lib/documents/sources/connections"; +import { DocumentSourceError } from "@/lib/documents/sources/http"; +import { evaluateSourceBaseUrl } from "@/lib/documents/sources/origins"; +import { + admitDocumentSourceCaller, + checkConnectRateLimit, + responseForSourceFailure, + sourceErrorResponse, + systemParam, +} from "@/lib/documents/sources/route-support"; +import { annotate } from "@/lib/logging/context"; +import { rateLimitHeaders } from "@/lib/rate-limit"; +import { documentSourceSaveSchema } from "@/lib/validations/document-sources"; + +export const dynamic = "force-dynamic"; + +type RouteParams = { params: Promise<{ system: string }> }; + +export const PUT = apiHandler( + async (request: NextRequest, { params }: RouteParams) => { + const auth = await requireAuth(); + const refused = await admitDocumentSourceCaller(auth); + if (refused) return refused; + const system = await systemParam(params); + if (system instanceof Response) return system; + const userId = auth.user.id; + annotate({ action: { name: "documents.sources.save" }, meta: { system } }); + + const rl = await checkConnectRateLimit(userId); + if (!rl.allowed) { + return sourceErrorResponse( + "rateLimited", + undefined, + rateLimitHeaders(rl), + ); + } + + const { data: body, error: jsonError } = await safeJson(request, { + maxBytes: 8 * 1024, + }); + if (jsonError) return jsonError; + const parsed = documentSourceSaveSchema.safeParse(body); + if (!parsed.success) { + return apiValidationError( + "Invalid connection details", + sanitiseZodIssues(parsed.error.issues), + 422, + ); + } + + const verdict = evaluateSourceBaseUrl(parsed.data.baseUrl); + if (!verdict.ok) { + return verdict.reason === "notAllowed" + ? sourceErrorResponse("originNotAllowed") + : apiError("Enter the full address, starting with https://.", 422, { + errorCode: "documents.sources.invalidAddress", + }); + } + + const organizationId = + system === "PAPRA" ? (parsed.data.organizationId ?? null) : null; + if (system === "PAPRA" && !organizationId) { + return apiError("Papra needs the organization id.", 422, { + errorCode: "documents.sources.organizationRequired", + }); + } + + const existing = await loadConnection(userId, system); + const token = + parsed.data.token ?? (existing ? decrypt(existing.tokenEncrypted) : null); + if (!token) { + return apiError("Enter the API token.", 422, { + errorCode: "documents.sources.tokenRequired", + }); + } + + // Live check before anything is stored: the row only ever holds a + // connection that answered once. + try { + await sourceClient({ + system, + origin: verdict.origin, + baseUrl: verdict.baseUrl, + organizationId, + token, + }).test(); + } catch (err) { + if (err instanceof DocumentSourceError) { + annotate({ meta: { refused: err.code } }); + } + return responseForSourceFailure(err); + } + + const now = new Date(); + const tokenEncrypted = encrypt(token); + // Field by field; `userId` from the session, never the body. + const saved = await prisma.documentSourceConnection.upsert({ + where: { userId_system: { userId, system } }, + create: { + userId, + system, + baseUrl: verdict.baseUrl, + organizationId, + tokenEncrypted, + lastVerifiedAt: now, + }, + update: { + baseUrl: verdict.baseUrl, + organizationId, + tokenEncrypted, + lastVerifiedAt: now, + }, + select: { + system: true, + baseUrl: true, + organizationId: true, + lastVerifiedAt: true, + }, + }); + + await auditLog("documents.sources.connect", { + userId, + ipAddress: getClientIp(request), + details: { system, origin: verdict.origin, replaced: existing !== null }, + }); + + return apiSuccess(toConnectionDto(saved)); + }, +); + +export const DELETE = apiHandler( + async (request: NextRequest, { params }: RouteParams) => { + const auth = await requireAuth(); + // No list needed to forget a connection: an operator who removed the + // origin must not leave the person unable to delete the stored token. + const refused = await admitDocumentSourceCaller(auth, { needsList: false }); + if (refused) return refused; + const system = await systemParam(params); + if (system instanceof Response) return system; + annotate({ + action: { name: "documents.sources.disconnect" }, + meta: { system }, + }); + + const { count } = await prisma.documentSourceConnection.deleteMany({ + where: { userId: auth.user.id, system }, + }); + if (count > 0) { + await auditLog("documents.sources.disconnect", { + userId: auth.user.id, + ipAddress: getClientIp(request), + details: { system }, + }); + } + return apiSuccess({ disconnected: count > 0 }); + }, +); diff --git a/src/app/api/documents/sources/[system]/search/route.ts b/src/app/api/documents/sources/[system]/search/route.ts new file mode 100644 index 000000000..7296048e8 --- /dev/null +++ b/src/app/api/documents/sources/[system]/search/route.ts @@ -0,0 +1,131 @@ +/** + * Document picker (#1038): search one connected document archive. + * + * `q` searches the document's name (the title in Paperless-ngx, the name in + * Papra), `tag` narrows to one of the source's tags by its id, `from` / `to` + * to a date range (inclusive), `page` pages through 25 at a time. The answer + * is a minimal row per document (source id, title, date, tag names, size when + * the source says it) plus what HealthLog already holds under that source key: + * `new`, `imported` (with the vault document's id) or `deleted` (the person + * deleted it here; importing it again is refused). Nothing is stored. + * + * Sixty searches a minute, shared with the tag read. Cookie-only and + * owner-only; see `admitDocumentSourceCaller`. + */ +import { NextRequest } from "next/server"; + +import { apiHandler, requireAuth } from "@/lib/api-handler"; +import { + apiSuccess, + apiValidationError, + sanitiseZodIssues, +} from "@/lib/api-response"; +import { findSourceKeyStates } from "@/lib/documents/source-key"; +import type { SourceListItem } from "@/lib/documents/sources/clients"; +import { openConnection } from "@/lib/documents/sources/connections"; +import { + admitDocumentSourceCaller, + checkSearchRateLimit, + responseForSourceFailure, + sourceErrorResponse, + systemParam, +} from "@/lib/documents/sources/route-support"; +import type { + DocumentSourceResultDto, + DocumentSourceSearchDto, +} from "@/lib/documents/sources/types"; +import { annotate } from "@/lib/logging/context"; +import { rateLimitHeaders } from "@/lib/rate-limit"; +import { documentSourceSearchSchema } from "@/lib/validations/document-sources"; + +export const dynamic = "force-dynamic"; + +type RouteParams = { params: Promise<{ system: string }> }; + +export const GET = apiHandler( + async (request: NextRequest, { params }: RouteParams) => { + const auth = await requireAuth(); + const refused = await admitDocumentSourceCaller(auth); + if (refused) return refused; + const system = await systemParam(params); + if (system instanceof Response) return system; + const userId = auth.user.id; + + const url = new URL(request.url); + const parsed = documentSourceSearchSchema.safeParse({ + q: url.searchParams.get("q") ?? undefined, + tag: url.searchParams.get("tag") ?? undefined, + from: url.searchParams.get("from") ?? undefined, + to: url.searchParams.get("to") ?? undefined, + page: url.searchParams.get("page") ?? undefined, + }); + if (!parsed.success) { + return apiValidationError( + "Invalid search", + sanitiseZodIssues(parsed.error.issues), + 422, + ); + } + const { q, tag, from, to, page } = parsed.data; + + const rl = await checkSearchRateLimit(userId); + if (!rl.allowed) { + return sourceErrorResponse( + "rateLimited", + undefined, + rateLimitHeaders(rl), + ); + } + + let found: { items: SourceListItem[]; hasMore: boolean }; + try { + const client = await openConnection(userId, system); + found = await client.search({ + q, + tagId: tag ?? null, + from: from ?? null, + to: to ?? null, + page, + }); + } catch (err) { + return responseForSourceFailure(err); + } + + const states = await findSourceKeyStates( + userId, + system, + found.items.map((item) => item.sourceId), + ); + const results: DocumentSourceResultDto[] = found.items.map((item) => { + const held = states.get(item.sourceId); + return { + sourceId: item.sourceId, + title: item.title, + date: item.date, + tags: item.tags, + sizeBytes: item.sizeBytes, + state: held?.state ?? "new", + documentId: held?.state === "imported" ? held.documentId : null, + }; + }); + + // Counts only: the query and the titles are the person's own words and + // stay out of the wide event. + annotate({ + action: { name: "documents.sources.search" }, + meta: { + system, + page, + count: results.length, + filtered: Boolean(q || tag || from || to), + }, + }); + + const body: DocumentSourceSearchDto = { + results, + page, + hasMore: found.hasMore, + }; + return apiSuccess(body); + }, +); diff --git a/src/app/api/documents/sources/[system]/tags/route.ts b/src/app/api/documents/sources/[system]/tags/route.ts new file mode 100644 index 000000000..4b788608c --- /dev/null +++ b/src/app/api/documents/sources/[system]/tags/route.ts @@ -0,0 +1,52 @@ +/** + * Document picker (#1038): the tags of one connected document archive, for the + * picker's tag filter. The first 250 by name, as `{ id, name }` in the + * source's own ids. Shares the search allowance (60 a minute). + * + * Cookie-only and owner-only; see `admitDocumentSourceCaller`. + */ +import { NextRequest } from "next/server"; + +import { apiHandler, requireAuth } from "@/lib/api-handler"; +import { apiSuccess } from "@/lib/api-response"; +import { openConnection } from "@/lib/documents/sources/connections"; +import { + admitDocumentSourceCaller, + checkSearchRateLimit, + responseForSourceFailure, + sourceErrorResponse, + systemParam, +} from "@/lib/documents/sources/route-support"; +import { annotate } from "@/lib/logging/context"; +import { rateLimitHeaders } from "@/lib/rate-limit"; + +export const dynamic = "force-dynamic"; + +type RouteParams = { params: Promise<{ system: string }> }; + +export const GET = apiHandler( + async (_request: NextRequest, { params }: RouteParams) => { + const auth = await requireAuth(); + const refused = await admitDocumentSourceCaller(auth); + if (refused) return refused; + const system = await systemParam(params); + if (system instanceof Response) return system; + annotate({ action: { name: "documents.sources.tags" }, meta: { system } }); + + const rl = await checkSearchRateLimit(auth.user.id); + if (!rl.allowed) { + return sourceErrorResponse( + "rateLimited", + undefined, + rateLimitHeaders(rl), + ); + } + + try { + const client = await openConnection(auth.user.id, system); + return apiSuccess({ tags: await client.tags() }); + } catch (err) { + return responseForSourceFailure(err); + } + }, +); diff --git a/src/app/api/documents/sources/[system]/test/route.ts b/src/app/api/documents/sources/[system]/test/route.ts new file mode 100644 index 000000000..536b0c7d1 --- /dev/null +++ b/src/app/api/documents/sources/[system]/test/route.ts @@ -0,0 +1,71 @@ +/** + * Document picker (#1038): test the saved connection to one document archive. + * + * One request to the source with the stored token, answered `{ ok, latencyMs }` + * or with the reason it failed (`meta.errorCode`, and `meta.upstreamStatus` + * when the source answered at all; its body is never passed on). A success + * stamps `lastVerifiedAt`. Shares the save's allowance of ten a minute. + * + * Cookie-only and owner-only; see `admitDocumentSourceCaller`. + */ +import { NextRequest } from "next/server"; + +import { apiHandler, requireAuth } from "@/lib/api-handler"; +import { apiSuccess } from "@/lib/api-response"; +import { + markVerified, + openConnection, +} from "@/lib/documents/sources/connections"; +import { DocumentSourceError } from "@/lib/documents/sources/http"; +import { + admitDocumentSourceCaller, + checkConnectRateLimit, + responseForSourceFailure, + sourceErrorResponse, + systemParam, +} from "@/lib/documents/sources/route-support"; +import { annotate } from "@/lib/logging/context"; +import { rateLimitHeaders } from "@/lib/rate-limit"; + +export const dynamic = "force-dynamic"; + +type RouteParams = { params: Promise<{ system: string }> }; + +export const POST = apiHandler( + async (_request: NextRequest, { params }: RouteParams) => { + const auth = await requireAuth(); + const refused = await admitDocumentSourceCaller(auth); + if (refused) return refused; + const system = await systemParam(params); + if (system instanceof Response) return system; + const userId = auth.user.id; + annotate({ action: { name: "documents.sources.test" }, meta: { system } }); + + const rl = await checkConnectRateLimit(userId); + if (!rl.allowed) { + return sourceErrorResponse( + "rateLimited", + undefined, + rateLimitHeaders(rl), + ); + } + + const start = performance.now(); + try { + const client = await openConnection(userId, system); + await client.test(); + } catch (err) { + if (err instanceof DocumentSourceError) { + annotate({ meta: { refused: err.code } }); + } + return responseForSourceFailure(err); + } + const latencyMs = Math.round(performance.now() - start); + const verifiedAt = await markVerified(userId, system); + return apiSuccess({ + ok: true, + latencyMs, + lastVerifiedAt: verifiedAt.toISOString(), + }); + }, +); diff --git a/src/app/api/documents/sources/route.ts b/src/app/api/documents/sources/route.ts new file mode 100644 index 000000000..0a91b184b --- /dev/null +++ b/src/app/api/documents/sources/route.ts @@ -0,0 +1,54 @@ +/** + * Document picker (#1038): which document archives this person can pick from. + * + * `available` says whether the operator listed any origin in + * `DOCUMENT_SOURCE_ORIGINS`; `allowedOrigins` lists them so the settings card + * can say which addresses will be accepted; `connections` are the person's own + * saved connections, never with their token (`hasToken` is the literal true). + * No request leaves the server here. + * + * Cookie-only and owner-only like every route under this path + * (`admitDocumentSourceCaller`). With the list unset it answers 200 + * `available: false` rather than 404, because this is the one read the client + * makes to decide whether to show the picker at all. + */ +import { apiHandler, requireAuth } from "@/lib/api-handler"; +import { apiSuccess } from "@/lib/api-response"; +import { + listConnections, + toConnectionDto, +} from "@/lib/documents/sources/connections"; +import { + documentSourcesAvailable, + listedDocumentSourceOrigins, +} from "@/lib/documents/sources/origins"; +import type { DocumentSourcesStatusDto } from "@/lib/documents/sources/types"; +import { admitDocumentSourceCaller } from "@/lib/documents/sources/route-support"; +import { annotate } from "@/lib/logging/context"; + +export const dynamic = "force-dynamic"; + +export const GET = apiHandler(async () => { + const auth = await requireAuth(); + const refused = await admitDocumentSourceCaller(auth, { needsList: false }); + if (refused) return refused; + + annotate({ action: { name: "documents.sources.status" } }); + + if (!documentSourcesAvailable()) { + const off: DocumentSourcesStatusDto = { + available: false, + allowedOrigins: [], + connections: [], + }; + return apiSuccess(off); + } + + const rows = await listConnections(auth.user.id); + const status: DocumentSourcesStatusDto = { + available: true, + allowedOrigins: listedDocumentSourceOrigins(), + connections: rows.map(toConnectionDto), + }; + return apiSuccess(status); +}); diff --git a/src/lib/crypto/encrypted-columns.ts b/src/lib/crypto/encrypted-columns.ts index 684228dba..6759e5d7e 100644 --- a/src/lib/crypto/encrypted-columns.ts +++ b/src/lib/crypto/encrypted-columns.ts @@ -161,6 +161,14 @@ export const ENCRYPTED_COLUMNS: readonly EncryptedColumn[] = [ { model: "GoogleHealthConnection", field: "accessToken", kind: "string" }, { model: "GoogleHealthConnection", field: "refreshToken", kind: "string" }, + // ───── Document picker (v1.39.3, #1038) ───── + // A person's API token for their Paperless-ngx or Papra instance. + { + model: "DocumentSourceConnection", + field: "tokenEncrypted", + kind: "string", + }, + // ───── AppSettings — operator credentials ───── { model: "AppSettings", field: "adminAiKeyEncrypted", kind: "string" }, { diff --git a/src/lib/data-wipe/wipe-plan.ts b/src/lib/data-wipe/wipe-plan.ts index 3bb1bcf60..1e4f2ff9e 100644 --- a/src/lib/data-wipe/wipe-plan.ts +++ b/src/lib/data-wipe/wipe-plan.ts @@ -155,6 +155,10 @@ export const WIPE_MODELS = [ // Further import keys that resolved to a document by content; a child of // the document, removed before it so the counts are truthful. "DocumentSourceAlias", + // The person's own connections to Paperless-ngx / Papra (#1038), with their + // encrypted API tokens. A wipe that left them would leave a working + // credential to another system behind an account that erased everything. + "DocumentSourceConnection", "ExtractedFact", "ImportJob", diff --git a/src/lib/documents/source-key.ts b/src/lib/documents/source-key.ts index 0c5656f5b..61fa6580d 100644 --- a/src/lib/documents/source-key.ts +++ b/src/lib/documents/source-key.ts @@ -107,3 +107,64 @@ export function checkSourceLookupRateLimit( SOURCE_LOOKUP_WINDOW_MS, ); } + +/** What the vault holds for one source key, as the picker lists it. */ +export type SourceKeyState = + { state: "imported"; documentId: string } | { state: "deleted" }; + +/** + * {@link findSourceKey} for a page of results at once: the same three places, + * three queries for the whole page instead of three per result. A key held by + * a live document answers `imported` with its id; one held by a tombstone, or + * kept by the purge ledger, answers `deleted`. A key the vault does not know + * is absent from the map. + */ +export async function findSourceKeyStates( + userId: string, + sourceSystem: DocumentSourceSystemValue, + sourceIds: string[], +): Promise> { + const states = new Map(); + const ids = [...new Set(sourceIds)]; + if (ids.length === 0) return states; + + const [own, aliases, purged] = await Promise.all([ + prisma.inboundDocument.findMany({ + where: { userId, sourceSystem, sourceId: { in: ids } }, + select: { id: true, sourceId: true, deletedAt: true }, + }), + prisma.documentSourceAlias.findMany({ + where: { userId, sourceSystem, sourceId: { in: ids } }, + select: { + sourceId: true, + document: { select: { id: true, deletedAt: true } }, + }, + }), + prisma.documentImportKey.findMany({ + where: { userId, sourceSystem, sourceId: { in: ids } }, + select: { sourceId: true }, + }), + ]); + + // Lowest precedence first, so the document's own key wins, as it does in + // `findSourceKey`. + for (const row of purged) states.set(row.sourceId, { state: "deleted" }); + for (const row of aliases) { + states.set( + row.sourceId, + row.document.deletedAt + ? { state: "deleted" } + : { state: "imported", documentId: row.document.id }, + ); + } + for (const row of own) { + if (!row.sourceId) continue; + states.set( + row.sourceId, + row.deletedAt + ? { state: "deleted" } + : { state: "imported", documentId: row.id }, + ); + } + return states; +} diff --git a/src/lib/documents/sources/clients.ts b/src/lib/documents/sources/clients.ts new file mode 100644 index 000000000..7bca6ed92 --- /dev/null +++ b/src/lib/documents/sources/clients.ts @@ -0,0 +1,452 @@ +/** + * The two source adapters the document picker speaks: Paperless-ngx and Papra. + * + * Each turns the picker's four questions (does this connection work, which + * tags exist, which documents match, give me this one) into the source's own + * REST calls, and each answer into the picker's minimal shapes. Everything + * leaves through `./http.ts`, so the origin pin, the redirect refusal, the + * timeouts and the byte caps are not something an adapter can forget. + * + * Both adapters re-check tag and date filters on what comes back. A server + * that ignores a parameter (an older Paperless, a Papra whose search syntax + * changed) must not widen a result into documents the person filtered out; + * the import script makes the same check for the same reason. + * + * API versions: Paperless-ngx is asked for API version 9 (Paperless-ngx 2.16 + * and later, the version where `created` became a plain date; it answers 406 + * to a version it does not speak). Papra's endpoints are the documented + * organization-scoped ones (`/api/organizations/{id}/documents`, `/tags`, + * `/documents/{id}`, `/documents/{id}/file`). + */ +import { DOCUMENT_TITLE_MAX } from "@/lib/validations/inbound-documents"; + +import { + DocumentSourceError, + sourceDownload, + sourceJson, + type SourceTarget, +} from "./http"; +import { + DOCUMENT_PICKER_MAX_TAGS, + DOCUMENT_PICKER_PAGE_SIZE, + type DocumentPickerSystem, + type DocumentSourceTagDto, +} from "./types"; + +/** Paperless-ngx API version the picker speaks (Paperless-ngx 2.16+). */ +export const PAPERLESS_API_VERSION = 9; + +export interface SourceSearchParams { + /** Name to search for; empty lists everything. */ + q: string; + /** A tag id as the source names it. */ + tagId: string | null; + /** YYYY-MM-DD, inclusive. */ + from: string | null; + to: string | null; + /** 1-based. */ + page: number; +} + +export interface SourceListItem { + sourceId: string; + title: string; + date: string | null; + tags: string[]; + sizeBytes: number | null; +} + +export interface SourceDocumentMeta { + title: string; + date: string | null; + filename: string | null; +} + +export interface SourceClient { + readonly system: DocumentPickerSystem; + test(): Promise; + tags(): Promise; + search( + params: SourceSearchParams, + ): Promise<{ items: SourceListItem[]; hasMore: boolean }>; + document(sourceId: string): Promise; + download( + sourceId: string, + maxFileBytes: number, + ): Promise<{ bytes: Uint8Array; filename: string | null }>; +} + +export interface SourceConnection { + system: DocumentPickerSystem; + origin: string; + baseUrl: string; + organizationId: string | null; + token: string; +} + +// ── Defensive readers ────────────────────────────────────────────────────── + +type Json = Record; + +function asRecord(value: unknown): Json | null { + return value && typeof value === "object" && !Array.isArray(value) + ? (value as Json) + : null; +} + +function asArray(value: unknown): unknown[] { + return Array.isArray(value) ? value : []; +} + +function asText(value: unknown): string | null { + if (typeof value === "string") return value; + if (typeof value === "number" && Number.isFinite(value)) return String(value); + return null; +} + +/** The body is not the shape the API documents: most often a base address + * that points at the web app instead of the instance. */ +function shapeError(): DocumentSourceError { + return new DocumentSourceError("badResponse"); +} + +/** YYYY-MM-DD from an ISO date or date-time, or null. */ +export function dayOf(value: unknown): string | null { + if (typeof value !== "string") return null; + const day = value.slice(0, 10); + return /^\d{4}-\d{2}-\d{2}$/.test(day) ? day : null; +} + +function stripExtension(name: string | null): string { + return name ? name.replace(/\.[A-Za-z0-9]{1,5}$/, "") : ""; +} + +/** + * A title that is not blank and fits the vault: the source's title, else its + * file name without the extension, else a placeholder naming the source id. + */ +export function titleOf( + title: string | null, + filename: string | null, + fallback: string, +): string { + const picked = + (title ?? "").trim() || stripExtension(filename).trim() || fallback; + return picked.slice(0, DOCUMENT_TITLE_MAX); +} + +/** Is `day` inside the inclusive range? A document without a date is kept + * only when no range was asked for. */ +export function withinRange( + day: string | null, + from: string | null, + to: string | null, +): boolean { + if (!from && !to) return true; + if (!day) return false; + if (from && day < from) return false; + if (to && day > to) return false; + return true; +} + +// ── Paperless-ngx ────────────────────────────────────────────────────────── + +function paperless(connection: SourceConnection): SourceClient { + const target: SourceTarget = { + origin: connection.origin, + baseUrl: connection.baseUrl, + headers: { + Authorization: `Token ${connection.token}`, + Accept: `application/json; version=${PAPERLESS_API_VERSION}`, + }, + }; + const download: SourceTarget = { + ...target, + headers: { Authorization: `Token ${connection.token}` }, + }; + + /** A server that says which API version it speaks, and it is too old. */ + function checkVersion(response: Response): void { + const raw = response.headers.get("x-api-version"); + if (raw === null) return; + const version = Number(raw); + if (Number.isFinite(version) && version < PAPERLESS_API_VERSION) { + throw new DocumentSourceError("versionTooOld", response.status); + } + } + + async function tagMap(): Promise> { + const tags = await listTags(); + return new Map(tags.map((tag) => [tag.id, tag.name])); + } + + async function listTags(): Promise { + const query = new URLSearchParams({ + page_size: String(DOCUMENT_PICKER_MAX_TAGS), + ordering: "name", + }); + const { body, response } = await sourceJson(target, `/api/tags/?${query}`); + checkVersion(response); + const page = asRecord(body); + if (!page) throw shapeError(); + const tags: DocumentSourceTagDto[] = []; + for (const raw of asArray(page.results)) { + const tag = asRecord(raw); + const id = asText(tag?.id); + const name = asText(tag?.name); + if (id && name) tags.push({ id, name }); + } + return tags; + } + + return { + system: "PAPERLESS", + + async test() { + const { body, response } = await sourceJson( + target, + "/api/documents/?page_size=1&fields=id", + ); + checkVersion(response); + if (!asRecord(body)) throw shapeError(); + }, + + tags: listTags, + + async search(params) { + const query = new URLSearchParams({ + page: String(params.page), + page_size: String(DOCUMENT_PICKER_PAGE_SIZE), + ordering: "-created", + truncate_content: "true", + fields: "id,title,created,added,tags,original_file_name", + }); + if (params.q) query.set("title__icontains", params.q); + if (params.tagId) query.set("tags__id__all", params.tagId); + if (params.from) query.set("created__date__gte", params.from); + if (params.to) query.set("created__date__lte", params.to); + + let page: Json | null; + let names: Map; + try { + const [list, tags] = await Promise.all([ + sourceJson(target, `/api/documents/?${query}`), + tagMap(), + ]); + checkVersion(list.response); + page = asRecord(list.body); + names = tags; + } catch (err) { + // Paperless answers a page past the end with 404 ("Invalid page"): + // there is simply nothing more. + if (err instanceof DocumentSourceError && err.code === "notFound") { + return { items: [], hasMore: false }; + } + throw err; + } + if (!page) throw shapeError(); + + const items: SourceListItem[] = []; + for (const raw of asArray(page.results)) { + const doc = asRecord(raw); + const id = asText(doc?.id); + if (!doc || !id) continue; + const tagIds = asArray(doc.tags) + .map(asText) + .filter((t): t is string => t !== null); + if (params.tagId && !tagIds.includes(params.tagId)) continue; + const date = dayOf(doc.created) ?? dayOf(doc.added); + if (!withinRange(date, params.from, params.to)) continue; + items.push({ + sourceId: id, + title: titleOf( + asText(doc.title), + asText(doc.original_file_name), + `Paperless ${id}`, + ), + date, + tags: tagIds + .map((tagId) => names.get(tagId)) + .filter((n): n is string => typeof n === "string"), + sizeBytes: null, + }); + } + return { items, hasMore: typeof page.next === "string" && !!page.next }; + }, + + async document(sourceId) { + const { body, response } = await sourceJson( + target, + `/api/documents/${encodeURIComponent(sourceId)}/?fields=id,title,created,added,original_file_name`, + ); + checkVersion(response); + const doc = asRecord(body); + if (!doc) throw shapeError(); + const filename = asText(doc.original_file_name); + return { + title: titleOf(asText(doc.title), filename, `Paperless ${sourceId}`), + date: dayOf(doc.created) ?? dayOf(doc.added), + filename, + }; + }, + + download(sourceId, maxFileBytes) { + // The original, not the archived PDF/A: the same bytes a Paperless + // workflow or the import script sends, so they dedup against each other. + return sourceDownload( + download, + `/api/documents/${encodeURIComponent(sourceId)}/download/?original=true`, + maxFileBytes, + ); + }, + }; +} + +// ── Papra ────────────────────────────────────────────────────────────────── + +/** Papra's search syntax quotes a tag name that holds a space. */ +function papraTagTerm(name: string): string { + return /[\s"]/.test(name) ? `tag:${JSON.stringify(name)}` : `tag:${name}`; +} + +function papra(connection: SourceConnection): SourceClient { + if (!connection.organizationId) { + throw new DocumentSourceError("notFound"); + } + const org = `/api/organizations/${encodeURIComponent(connection.organizationId)}`; + const target: SourceTarget = { + origin: connection.origin, + baseUrl: connection.baseUrl, + headers: { + Authorization: `Bearer ${connection.token}`, + Accept: "application/json", + }, + }; + const download: SourceTarget = { + ...target, + headers: { Authorization: `Bearer ${connection.token}` }, + }; + + async function listTags(): Promise { + const { body } = await sourceJson(target, `${org}/tags`); + const page = asRecord(body); + if (!page) throw shapeError(); + const tags: DocumentSourceTagDto[] = []; + for (const raw of asArray(page.tags)) { + const tag = asRecord(raw); + const id = asText(tag?.id); + const name = asText(tag?.name); + if (id && name) tags.push({ id, name }); + } + return tags + .sort((a, b) => a.name.localeCompare(b.name)) + .slice(0, DOCUMENT_PICKER_MAX_TAGS); + } + + return { + system: "PAPRA", + + async test() { + const { body } = await sourceJson( + target, + `${org}/documents?pageIndex=0&pageSize=1`, + ); + if (!asRecord(body)) throw shapeError(); + }, + + tags: listTags, + + async search(params) { + const terms: string[] = []; + if (params.q) terms.push(params.q); + if (params.tagId) { + const tag = (await listTags()).find((t) => t.id === params.tagId); + // A tag that no longer exists matches nothing, rather than silently + // dropping the filter and listing everything. + if (!tag) return { items: [], hasMore: false }; + terms.push(papraTagTerm(tag.name)); + } + const pageIndex = params.page - 1; + const query = new URLSearchParams({ + pageIndex: String(pageIndex), + pageSize: String(DOCUMENT_PICKER_PAGE_SIZE), + }); + if (terms.length > 0) query.set("searchQuery", terms.join(" ")); + const { body } = await sourceJson(target, `${org}/documents?${query}`); + const page = asRecord(body); + if (!page) throw shapeError(); + + const documents = asArray(page.documents); + const items: SourceListItem[] = []; + for (const raw of documents) { + const doc = asRecord(raw); + const id = asText(doc?.id); + if (!doc || !id) continue; + const tags = asArray(doc.tags) + .map(asRecord) + .filter((t): t is Json => t !== null); + if (params.tagId && !tags.some((t) => asText(t.id) === params.tagId)) { + continue; + } + // Papra's search date syntax differs between versions; the range is + // applied here instead of trusting a query syntax. + const date = dayOf(doc.documentDate) ?? dayOf(doc.createdAt); + if (!withinRange(date, params.from, params.to)) continue; + const size = doc.originalSize; + items.push({ + sourceId: id, + title: titleOf( + stripExtension(asText(doc.name)), + asText(doc.originalName), + `Papra ${id}`, + ), + date, + tags: tags + .map((t) => asText(t.name)) + .filter((n): n is string => n !== null), + sizeBytes: typeof size === "number" && size >= 0 ? size : null, + }); + } + const total = + typeof page.documentsCount === "number" ? page.documentsCount : null; + const hasMore = + total !== null + ? (pageIndex + 1) * DOCUMENT_PICKER_PAGE_SIZE < total + : documents.length >= DOCUMENT_PICKER_PAGE_SIZE; + return { items, hasMore }; + }, + + async document(sourceId) { + const { body } = await sourceJson( + target, + `${org}/documents/${encodeURIComponent(sourceId)}`, + ); + const doc = asRecord(asRecord(body)?.document); + if (!doc) throw shapeError(); + const filename = asText(doc.originalName) ?? asText(doc.name); + return { + title: titleOf( + stripExtension(asText(doc.name)), + filename, + `Papra ${sourceId}`, + ), + date: dayOf(doc.documentDate) ?? dayOf(doc.createdAt), + filename, + }; + }, + + download(sourceId, maxFileBytes) { + return sourceDownload( + download, + `${org}/documents/${encodeURIComponent(sourceId)}/file`, + maxFileBytes, + ); + }, + }; +} + +export function sourceClient(connection: SourceConnection): SourceClient { + return connection.system === "PAPERLESS" + ? paperless(connection) + : papra(connection); +} diff --git a/src/lib/documents/sources/connections.ts b/src/lib/documents/sources/connections.ts new file mode 100644 index 000000000..c8f0dad2c --- /dev/null +++ b/src/lib/documents/sources/connections.ts @@ -0,0 +1,117 @@ +/** + * Reading a person's document-source connection, and the one place its token + * is decrypted. + * + * {@link toConnectionDto} is the only shape a connection leaves the server in, + * and it has no field that could carry the token: `hasToken` is the literal + * `true`. {@link openConnection} decrypts the token for exactly one request to + * the source and re-checks the stored origin against the operator's list on + * the way, so a connection whose origin was removed from + * `DOCUMENT_SOURCE_ORIGINS` answers `originNotAllowed` instead of dialling. + */ +import { decrypt } from "@/lib/crypto"; +import { prisma } from "@/lib/db"; + +import { sourceClient, type SourceClient } from "./clients"; +import { DocumentSourceError } from "./http"; +import { evaluateSourceBaseUrl } from "./origins"; +import type { + DocumentPickerSystem, + DocumentSourceConnectionDto, +} from "./types"; + +/** The columns a DTO or a request needs; never the whole row by accident. */ +const CONNECTION_SELECT = { + system: true, + baseUrl: true, + organizationId: true, + tokenEncrypted: true, + lastVerifiedAt: true, +} as const; + +export interface StoredConnection { + system: string; + baseUrl: string; + organizationId: string | null; + tokenEncrypted: string; + lastVerifiedAt: Date | null; +} + +export function toConnectionDto( + row: Omit, +): DocumentSourceConnectionDto { + return { + system: row.system as DocumentPickerSystem, + baseUrl: row.baseUrl, + organizationId: row.organizationId, + hasToken: true, + lastVerifiedAt: row.lastVerifiedAt?.toISOString() ?? null, + originAllowed: evaluateSourceBaseUrl(row.baseUrl).ok, + }; +} + +export function listConnections( + userId: string, +): Promise[]> { + return prisma.documentSourceConnection.findMany({ + where: { userId }, + select: { + system: true, + baseUrl: true, + organizationId: true, + lastVerifiedAt: true, + }, + orderBy: { system: "asc" }, + }); +} + +export function loadConnection( + userId: string, + system: DocumentPickerSystem, +): Promise { + return prisma.documentSourceConnection.findUnique({ + where: { userId_system: { userId, system } }, + select: CONNECTION_SELECT, + }); +} + +/** + * A client for one of the person's stored connections, or the reason there is + * none. Throws {@link DocumentSourceError}: `notConnected` when nothing is + * stored for the system, `originNotAllowed` when the operator no longer lists + * the stored origin. + */ +export async function openConnection( + userId: string, + system: DocumentPickerSystem, +): Promise { + const row = await loadConnection(userId, system); + if (!row) throw new DocumentSourceError("notConnected"); + return clientFor(row); +} + +/** A client for a row already in hand, with the origin re-checked. */ +export function clientFor(row: StoredConnection): SourceClient { + const verdict = evaluateSourceBaseUrl(row.baseUrl); + if (!verdict.ok) throw new DocumentSourceError("originNotAllowed"); + return sourceClient({ + system: row.system as DocumentPickerSystem, + origin: verdict.origin, + baseUrl: verdict.baseUrl, + organizationId: row.organizationId, + token: decrypt(row.tokenEncrypted), + }); +} + +/** Stamp a successful test. */ +export async function markVerified( + userId: string, + system: DocumentPickerSystem, +): Promise { + const at = new Date(); + await prisma.documentSourceConnection.updateMany({ + where: { userId, system }, + data: { lastVerifiedAt: at }, + }); + return at; +} diff --git a/src/lib/documents/sources/http.ts b/src/lib/documents/sources/http.ts new file mode 100644 index 000000000..473d442ea --- /dev/null +++ b/src/lib/documents/sources/http.ts @@ -0,0 +1,198 @@ +/** + * The document picker's one way to the network. + * + * Every request to a Paperless-ngx or Papra instance goes through + * {@link sourceRequest}, which goes through `safeFetch` with the connection's + * origin as `operatorApprovedPrivateOrigin`: + * + * - the target must have exactly that origin (a path, query or `next` link + * that names another host is refused before a socket opens); + * - redirects are never followed; a 3xx is reported as `redirected`, so a + * source cannot bounce the request and its `Authorization` header to + * another host; + * - the connect goes through the pinned operator-approved dispatcher, which + * refuses link-local, metadata and unspecified addresses at dial time even + * for a listed name; + * - every call has a timeout, and every body is read with a byte cap. + * + * The origin handed in is the one the caller just re-checked against + * `DOCUMENT_SOURCE_ORIGINS`. Nothing here decides trust; it only refuses to + * go anywhere else. + */ +import { safeFetch, SafeFetchError } from "@/lib/safe-fetch"; +import { BodyTooLargeError, readBoundedBody } from "@/lib/labs/ocr-upload"; +import { annotate } from "@/lib/logging/context"; + +import type { DocumentSourceErrorCode } from "./types"; + +/** JSON bodies larger than this are not a search page; refuse them. */ +export const SOURCE_JSON_MAX_BYTES = 1024 * 1024; +export const SOURCE_JSON_TIMEOUT_MS = 10_000; +export const SOURCE_DOWNLOAD_TIMEOUT_MS = 60_000; + +/** + * A request to a source ended somewhere other than a usable answer. The code + * is what the client translates; `upstreamStatus` is the only thing about the + * source's answer that is passed on. Its body never is: an error page can + * echo the request's credentials or internal paths. + */ +export class DocumentSourceError extends Error { + readonly code: DocumentSourceErrorCode; + readonly upstreamStatus?: number; + + constructor(code: DocumentSourceErrorCode, upstreamStatus?: number) { + super(`document source: ${code}`); + this.name = "DocumentSourceError"; + this.code = code; + if (upstreamStatus !== undefined) this.upstreamStatus = upstreamStatus; + } +} + +/** The document is larger than the vault accepts; carries the cap. */ +export class DocumentSourceTooLargeError extends Error { + constructor(readonly maxFileBytes: number) { + super("document source: file larger than the vault accepts"); + this.name = "DocumentSourceTooLargeError"; + } +} + +export interface SourceTarget { + /** The listed origin, exactly as `DOCUMENT_SOURCE_ORIGINS` holds it. */ + origin: string; + /** Origin plus the optional base path, no trailing slash. */ + baseUrl: string; + headers: Record; +} + +async function sourceRequest( + target: SourceTarget, + path: string, + timeoutMs: number, +): Promise { + const url = `${target.baseUrl}${path}`; + let response: Response; + try { + response = await safeFetch( + url, + { method: "GET", headers: target.headers, redirect: "manual" }, + { operatorApprovedPrivateOrigin: target.origin, timeoutMs }, + ); + } catch (err) { + // A refused destination, a timeout and a dead host all read the same to + // the person: the source could not be reached from here. The operator's + // wide event keeps the kind. + annotate({ + meta: { + sourceFailure: err instanceof SafeFetchError ? err.kind : "network", + }, + }); + throw new DocumentSourceError("unreachable"); + } + if (response.status >= 300 && response.status < 400) { + await discard(response); + throw new DocumentSourceError("redirected", response.status); + } + return response; +} + +async function discard(response: Response): Promise { + try { + await response.body?.cancel(); + } catch { + // nothing to release + } +} + +/** Map a non-2xx answer to the error the person sees. */ +export async function failFor( + response: Response, + notFound: DocumentSourceErrorCode = "notFound", +): Promise { + await discard(response); + const status = response.status; + if (status === 401 || status === 403) { + throw new DocumentSourceError("authRefused", status); + } + if (status === 404) throw new DocumentSourceError(notFound, status); + // Paperless-ngx answers 406 to an API version it no longer (or not yet) + // speaks. + if (status === 406) throw new DocumentSourceError("versionTooOld", status); + throw new DocumentSourceError("badResponse", status); +} + +/** GET a JSON document, bounded in time and size. */ +export async function sourceJson( + target: SourceTarget, + path: string, + options: { notFound?: DocumentSourceErrorCode } = {}, +): Promise<{ body: unknown; response: Response }> { + const response = await sourceRequest(target, path, SOURCE_JSON_TIMEOUT_MS); + if (!response.ok) return failFor(response, options.notFound); + const declared = Number(response.headers.get("content-length") ?? 0); + if (declared > SOURCE_JSON_MAX_BYTES) { + await discard(response); + throw new DocumentSourceError("badResponse", response.status); + } + let bytes: Uint8Array; + try { + bytes = await readBoundedBody(response.body, SOURCE_JSON_MAX_BYTES); + } catch { + throw new DocumentSourceError("badResponse", response.status); + } + try { + return { + body: JSON.parse(new TextDecoder().decode(bytes)) as unknown, + response, + }; + } catch { + // A login page or a proxy's HTML error instead of the API: most often a + // base address that points at the web app rather than its API root. + throw new DocumentSourceError("badResponse", response.status); + } +} + +/** GET a file, bounded by the vault's own per-file cap. */ +export async function sourceDownload( + target: SourceTarget, + path: string, + maxFileBytes: number, +): Promise<{ bytes: Uint8Array; filename: string | null }> { + const response = await sourceRequest( + target, + path, + SOURCE_DOWNLOAD_TIMEOUT_MS, + ); + if (!response.ok) return failFor(response); + const declared = Number(response.headers.get("content-length") ?? 0); + if (declared > maxFileBytes) { + await discard(response); + throw new DocumentSourceTooLargeError(maxFileBytes); + } + try { + const bytes = await readBoundedBody(response.body, maxFileBytes); + return { + bytes, + filename: filenameFrom(response.headers.get("content-disposition")), + }; + } catch (err) { + if (err instanceof BodyTooLargeError) { + throw new DocumentSourceTooLargeError(maxFileBytes); + } + throw new DocumentSourceError("unreachable"); + } +} + +/** The file name a download names, if it names one. */ +export function filenameFrom(header: string | null): string | null { + if (!header) return null; + const star = /filename\*=(?:UTF-8'')?([^;]+)/i.exec(header); + if (star) { + try { + return decodeURIComponent(star[1].trim().replace(/^"|"$/g, "")); + } catch { + // fall through to the plain form + } + } + const plain = /filename="?([^";]+)"?/i.exec(header); + return plain ? plain[1] : null; +} diff --git a/src/lib/documents/sources/origins.ts b/src/lib/documents/sources/origins.ts new file mode 100644 index 000000000..6d7bb16a2 --- /dev/null +++ b/src/lib/documents/sources/origins.ts @@ -0,0 +1,95 @@ +/** + * `DOCUMENT_SOURCE_ORIGINS` (#1038) — the operator grant that lets the + * document picker reach a Paperless-ngx or Papra instance. + * + * DOCUMENT_SOURCE_ORIGINS="https://paperless.example.com,http://papra.lan:1221" + * + * The grammar is the one every private-origin list shares + * (`src/lib/private-origin-policy.ts`): exact `scheme://host[:port]` origins, + * no path, query, credentials, wildcard, CIDR or suffix. A malformed entry is + * logged once, reduced to scheme and host, and grants nothing; the valid + * entries beside it keep working. + * + * Unlike the notification list, this one is the WHOLE trust decision, not an + * exception on top of a public-host default. An origin that is not listed is + * refused even when it is a public address, and with the variable unset the + * picker does not exist: the routes answer 404 and the client renders no + * button. A person can only ever point their connection at a system the + * operator named. Every request to a listed origin still goes through + * `safeFetch` with the origin pinned, redirects forbidden, and the dial-time + * floor that refuses link-local, metadata and unspecified addresses even for + * a listed name. + * + * Read directly from `process.env` so the compose-whitelist guard sees it, + * parsed once per distinct value, and re-read on every use: an operator who + * removes an origin cuts off the stored connections that name it on their + * next request. + */ +import { + parsePrivateOrigins, + redactGrantEntry, +} from "@/lib/private-origin-policy"; + +const ENV_NAME = "DOCUMENT_SOURCE_ORIGINS"; + +let cachedRaw: string | undefined; +let cachedOrigins: ReadonlySet = new Set(); + +export function configuredDocumentSourceOrigins(): ReadonlySet { + const raw = process.env.DOCUMENT_SOURCE_ORIGINS; + if (raw === cachedRaw) return cachedOrigins; + cachedOrigins = parsePrivateOrigins(raw, (redactedEntry, reason) => { + console.warn(`${ENV_NAME}: ignoring entry "${redactedEntry}": ${reason}`); + }); + cachedRaw = raw; + return cachedOrigins; +} + +/** Is the picker switched on for this instance at all? */ +export function documentSourcesAvailable(): boolean { + return configuredDocumentSourceOrigins().size > 0; +} + +/** Longest base URL a connection may store. */ +export const DOCUMENT_SOURCE_BASE_URL_MAX = 2048; + +export type SourceBaseVerdict = + | { ok: true; origin: string; baseUrl: string } + | { ok: false; reason: "invalid" | "notAllowed" }; + +/** + * Validate a base URL a person typed against the operator's list. + * + * The base may carry a path (a Paperless served under `/paperless`); only its + * origin is compared, and it must be listed exactly. Userinfo, a query and a + * fragment are refused: none of them belongs in a base address, and each is + * where a pasted credential would sit. Trailing slashes are trimmed so the + * client can append `/api/...` without doubling one. + */ +export function evaluateSourceBaseUrl( + value: string, + origins: ReadonlySet = configuredDocumentSourceOrigins(), +): SourceBaseVerdict { + let url: URL; + try { + url = new URL(value.trim()); + } catch { + return { ok: false, reason: "invalid" }; + } + if (url.protocol !== "http:" && url.protocol !== "https:") { + return { ok: false, reason: "invalid" }; + } + if (url.username || url.password || url.search || url.hash) { + return { ok: false, reason: "invalid" }; + } + if (!origins.has(url.origin)) return { ok: false, reason: "notAllowed" }; + const path = url.pathname.replace(/\/+$/, ""); + return { ok: true, origin: url.origin, baseUrl: `${url.origin}${path}` }; +} + +/** The listed origins as the settings card may show them. */ +export function listedDocumentSourceOrigins(): string[] { + return [...configuredDocumentSourceOrigins()] + .map((origin) => redactGrantEntry(origin)) + .sort(); +} diff --git a/src/lib/documents/sources/route-support.ts b/src/lib/documents/sources/route-support.ts new file mode 100644 index 000000000..295944b8c --- /dev/null +++ b/src/lib/documents/sources/route-support.ts @@ -0,0 +1,156 @@ +/** + * What every document-source route does before its own work, and how a source + * failure becomes a response. + * + * The admission rule, with {@link admitDocumentSourceCaller} as its second + * half: + * + * - the route's own `requireAuth()` with no scope: a session, or a `["*"]` token; a narrow + * token is refused, and so is any request acting on somebody else's record + * (403 `sharing.not_permitted`). A delegate, whatever their grant, and a + * guardian acting for a managed record never reach a connection: it is the + * owner's credential to a system outside HealthLog, and driving it would + * read the owner's whole archive there, which no grant covers; + * - then the transport must be the cookie. A `["*"]` Bearer (the native + * client, a script holding a login token) is refused: the picker is a + * browser surface, and a credential for another system is managed only + * from one; + * - the documents module must be on, and the operator must have listed at + * least one origin in `DOCUMENT_SOURCE_ORIGINS`. Without the list the + * routes answer 404 before reading a row. + */ +import type { AuthContext } from "@/lib/api-handler"; +import { apiError } from "@/lib/api-response"; +import { requireModuleEnabled } from "@/lib/modules/gate"; +import { checkRateLimit, type RateLimitResult } from "@/lib/rate-limit"; + +import { DocumentSourceError, DocumentSourceTooLargeError } from "./http"; +import { documentSourcesAvailable } from "./origins"; +import { + documentSourceErrorCode, + systemFromSlug, + type DocumentPickerSystem, + type DocumentSourceErrorCode, +} from "./types"; + +/** + * Admit a caller the route has already resolved with a bare `requireAuth()`. + * + * Each route calls `requireAuth()` itself, in its own module, so the sharing + * guard's scan sees every one of them as a bare, owner-only route rather than + * a helper it cannot follow. This adds the rest: a cookie transport, the + * documents module, and (unless `needsList` is false, which only the status + * read passes) the operator's list. + */ +export async function admitDocumentSourceCaller( + auth: AuthContext, + options: { needsList?: boolean } = {}, +): Promise { + if (auth.authMethod !== "cookie") { + return apiError( + "Document archives are managed from a browser session.", + 403, + { errorCode: "documents.sources.browserOnly" }, + ); + } + const gate = await requireModuleEnabled(auth.user.id, "inboundDocuments"); + if (!gate.enabled) return gate.response; + if (options.needsList !== false && !documentSourcesAvailable()) { + return sourceErrorResponse("unavailable"); + } + return null; +} + +/** The `{system}` path segment, or a 404 for anything else. */ +export async function systemParam( + params: Promise<{ system: string }>, +): Promise { + const { system } = await params; + const resolved = systemFromSlug(system); + return resolved ?? apiError("Unknown document system.", 404); +} + +const STATUS: Record = { + unavailable: 404, + notConnected: 404, + originNotAllowed: 422, + unreachable: 502, + redirected: 502, + authRefused: 502, + versionTooOld: 502, + badResponse: 502, + notFound: 404, + linkTargetNotFound: 404, + rateLimited: 429, +}; + +const MESSAGE: Record = { + unavailable: "Document archives are not enabled on this server.", + notConnected: "This document archive is not connected.", + originNotAllowed: + "This address is not on the server's list of allowed document archives.", + unreachable: "The document archive could not be reached.", + redirected: + "The document archive redirected the request. Use the address it redirects to.", + authRefused: "The document archive refused the API token.", + versionTooOld: "The document archive's version is too old.", + badResponse: "The document archive gave an answer HealthLog cannot read.", + notFound: "The document archive has no such document or organization.", + linkTargetNotFound: "The record to link to was not found.", + rateLimited: "Too many requests. Try again later.", +}; + +export function sourceErrorResponse( + code: DocumentSourceErrorCode, + upstreamStatus?: number, + headers?: Record, +): Response { + return apiError(MESSAGE[code], STATUS[code], { + errorCode: documentSourceErrorCode(code), + ...(upstreamStatus !== undefined ? { upstreamStatus } : {}), + ...(headers ? { headers } : {}), + }); +} + +/** Map a thrown source failure to its response; rethrow anything else. */ +export function responseForSourceFailure(err: unknown): Response { + if (err instanceof DocumentSourceError) { + return sourceErrorResponse(err.code, err.upstreamStatus); + } + if (err instanceof DocumentSourceTooLargeError) { + return apiError("File is too large.", 413, { + errorCode: "documents.inbound.fileTooLarge", + reason: "fileTooLarge", + maxFileBytes: err.maxFileBytes, + }); + } + throw err; +} + +/** Saves and tests share one allowance: each one dials the source. */ +const CONNECT_LIMIT_PER_MINUTE = 10; + +export function checkConnectRateLimit( + userId: string, +): Promise { + return checkRateLimit( + `documents-source-connect:${userId}`, + CONNECT_LIMIT_PER_MINUTE, + 60_000, + ); +} + +/** + * Searches and tag reads per minute. Each one is a request to the source; a + * debounced search field sends one per pause in typing, which this leaves + * ample room for. + */ +const SEARCH_LIMIT_PER_MINUTE = 60; + +export function checkSearchRateLimit(userId: string): Promise { + return checkRateLimit( + `documents-source-search:${userId}`, + SEARCH_LIMIT_PER_MINUTE, + 60_000, + ); +} diff --git a/src/lib/documents/sources/types.ts b/src/lib/documents/sources/types.ts new file mode 100644 index 000000000..187f670b3 --- /dev/null +++ b/src/lib/documents/sources/types.ts @@ -0,0 +1,127 @@ +/** + * Shared vocabulary of the document picker (#1038): which systems it speaks, + * the minimal shapes it hands to the client, and the errors a source can end + * in. A zero-dependency leaf so the client components, the OpenAPI registry + * and the routes import one list. + */ + +/** The systems the picker can search and import from. */ +export const DOCUMENT_PICKER_SYSTEMS = ["PAPERLESS", "PAPRA"] as const; +export type DocumentPickerSystem = (typeof DOCUMENT_PICKER_SYSTEMS)[number]; + +/** The lowercase path segment for each system (`/api/documents/sources/{slug}`). */ +export const DOCUMENT_PICKER_SLUGS = { + paperless: "PAPERLESS", + papra: "PAPRA", +} as const satisfies Record; +export type DocumentPickerSlug = keyof typeof DOCUMENT_PICKER_SLUGS; + +export function systemFromSlug( + slug: string | undefined, +): DocumentPickerSystem | null { + return slug && Object.hasOwn(DOCUMENT_PICKER_SLUGS, slug) + ? DOCUMENT_PICKER_SLUGS[slug as DocumentPickerSlug] + : null; +} + +export function slugForSystem( + system: DocumentPickerSystem, +): DocumentPickerSlug { + return system === "PAPERLESS" ? "paperless" : "papra"; +} + +/** Results per search page. */ +export const DOCUMENT_PICKER_PAGE_SIZE = 25; +/** Highest page a search may ask for. */ +export const DOCUMENT_PICKER_MAX_PAGE = 500; +/** Longest name query. */ +export const DOCUMENT_PICKER_QUERY_MAX = 200; +/** Documents the sheet imports in one run. */ +export const DOCUMENT_PICKER_MAX_SELECTION = 25; +/** Tags offered in the filter. */ +export const DOCUMENT_PICKER_MAX_TAGS = 250; + +/** A connection as every route may show it: never the token. */ +export interface DocumentSourceConnectionDto { + system: DocumentPickerSystem; + baseUrl: string; + organizationId: string | null; + hasToken: true; + lastVerifiedAt: string | null; + /** False once the operator removed the origin from the list. */ + originAllowed: boolean; +} + +export interface DocumentSourcesStatusDto { + /** The operator listed at least one origin; otherwise the picker is off. */ + available: boolean; + /** The listed origins, so the settings card can say what is allowed. */ + allowedOrigins: string[]; + connections: DocumentSourceConnectionDto[]; +} + +export interface DocumentSourceTagDto { + id: string; + name: string; +} + +/** Whether HealthLog already holds a result, by its source key. */ +export type DocumentPickerResultState = "new" | "imported" | "deleted"; + +export interface DocumentSourceResultDto { + sourceId: string; + title: string; + /** YYYY-MM-DD, the document's own date in the source, or null. */ + date: string | null; + tags: string[]; + sizeBytes: number | null; + state: DocumentPickerResultState; + /** The vault document when `state` is `imported`. */ + documentId: string | null; +} + +export interface DocumentSourceSearchDto { + results: DocumentSourceResultDto[]; + page: number; + hasMore: boolean; +} + +export type DocumentImportOutcome = "imported" | "duplicate" | "deleted"; + +export interface DocumentSourceImportDto { + outcome: DocumentImportOutcome; + /** The vault document; null only when a deleted one has been purged. */ + documentId: string | null; + /** The document was linked to the record the picker was opened from. */ + linked: boolean; +} + +/** Record kinds a picked document can be linked to on import. */ +export const DOCUMENT_PICKER_LINK_KINDS = [ + "conditionEpisode", + "encounter", + "vaccination", +] as const; +export type DocumentPickerLinkKind = + (typeof DOCUMENT_PICKER_LINK_KINDS)[number]; + +/** Why a request to a source failed, as `meta.errorCode` suffixes. */ +export const DOCUMENT_SOURCE_ERROR_CODES = [ + "unavailable", + "notConnected", + "originNotAllowed", + "unreachable", + "redirected", + "authRefused", + "versionTooOld", + "badResponse", + "notFound", + "linkTargetNotFound", + "rateLimited", +] as const; +export type DocumentSourceErrorCode = + (typeof DOCUMENT_SOURCE_ERROR_CODES)[number]; + +export function documentSourceErrorCode(code: DocumentSourceErrorCode): string { + return `documents.sources.${code}`; +} diff --git a/src/lib/export/backup-plan.ts b/src/lib/export/backup-plan.ts index c9fa7b80a..2d689337a 100644 --- a/src/lib/export/backup-plan.ts +++ b/src/lib/export/backup-plan.ts @@ -620,6 +620,8 @@ export const NOT_IN_BACKUP_MODELS: Readonly> = { "A CSRF nonce for a handshake that is valid for minutes and is already over by the time any backup is read.", McpOAuthConnection: "A grant issued to a client that was registered against this instance.", + DocumentSourceConnection: + "A person's API token for their Paperless-ngx or Papra instance. A backup file is not a credential store, same reasoning as ApiToken; and restored on another host it would point at an address that host's operator never allowed in DOCUMENT_SOURCE_ORIGINS. The documents imported through it are carried with the vault; reconnecting takes one form.", IntegrationStatus: "A ledger of what this deployment's syncs did. Restoring it onto another host would assert a sync history that host never had.", diff --git a/src/lib/validations/document-sources.ts b/src/lib/validations/document-sources.ts new file mode 100644 index 000000000..6997cb97d --- /dev/null +++ b/src/lib/validations/document-sources.ts @@ -0,0 +1,91 @@ +/** + * Request schemas of the document picker (#1038): saving a connection, + * searching a source, importing one picked document. No `userId` field in any + * of them; the owner is always the authenticated session. + */ +import { z } from "zod/v4"; + +import { DOCUMENT_SOURCE_BASE_URL_MAX } from "@/lib/documents/sources/origins"; +import { + DOCUMENT_PICKER_LINK_KINDS, + DOCUMENT_PICKER_MAX_PAGE, + DOCUMENT_PICKER_QUERY_MAX, +} from "@/lib/documents/sources/types"; +import { + documentSourceIdSchema, + INBOUND_DOCUMENT_KINDS, +} from "@/lib/validations/inbound-documents"; + +/** Printable ASCII without spaces: an id or a key, never prose. */ +const printable = /^[\x21-\x7e]+$/u; + +const isoDate = z + .string() + .regex(/^\d{4}-\d{2}-\d{2}$/u, "Expected a YYYY-MM-DD date"); + +/** Longest API token or key a connection stores. */ +export const DOCUMENT_SOURCE_TOKEN_MAX = 512; + +/** + * Save a connection. `token` may be left out when a connection for the system + * already exists: the stored one is kept, so the address or the organization + * can change without typing the key again. The first save must carry it (the + * route says so). `organizationId` is Papra's and required there. + */ +export const documentSourceSaveSchema = z.object({ + baseUrl: z.string().trim().min(1).max(DOCUMENT_SOURCE_BASE_URL_MAX), + organizationId: z + .string() + .trim() + .min(1) + .max(128) + .regex(printable, "Expected printable characters without spaces") + .optional(), + token: z + .string() + .trim() + .min(1) + .max(DOCUMENT_SOURCE_TOKEN_MAX) + .regex(printable, "Expected printable characters without spaces") + .optional(), +}); +export type DocumentSourceSaveInput = z.infer; + +export const documentSourceSearchSchema = z + .object({ + q: z.string().trim().max(DOCUMENT_PICKER_QUERY_MAX).default(""), + tag: z + .string() + .trim() + .min(1) + .max(64) + .regex(printable, "Expected printable characters without spaces") + .optional(), + from: isoDate.optional(), + to: isoDate.optional(), + page: z.coerce + .number() + .int() + .min(1) + .max(DOCUMENT_PICKER_MAX_PAGE) + .default(1), + }) + .refine((v) => !v.from || !v.to || v.from <= v.to, { + path: ["to"], + message: "The end date is before the start date", + }); + +export const documentSourceImportSchema = z.object({ + sourceId: documentSourceIdSchema, + kind: z.enum(INBOUND_DOCUMENT_KINDS).optional(), + /** The record the picker was opened from, to link the document to. */ + link: z + .object({ + kind: z.enum(DOCUMENT_PICKER_LINK_KINDS), + id: z.string().trim().min(1).max(64), + }) + .optional(), +}); +export type DocumentSourceImportInput = z.infer< + typeof documentSourceImportSchema +>; diff --git a/src/lib/validations/inbound-documents.ts b/src/lib/validations/inbound-documents.ts index cd78d826a..ebc8aa030 100644 --- a/src/lib/validations/inbound-documents.ts +++ b/src/lib/validations/inbound-documents.ts @@ -701,7 +701,7 @@ export const DOCUMENT_SOURCE_ID_MAX = 128; * The document's id in its source system. Printable ASCII only: it is a key, * not prose, and it is echoed on the detail sheet. */ -const documentSourceIdSchema = z +export const documentSourceIdSchema = z .string() .trim() .min(1)