From 89b18e001e0b63319d5af8335a5156c16bb126c1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Sat, 26 Sep 2026 12:53:32 +0200 Subject: [PATCH 01/18] refactor(documents): move the upload's store path into one ingest function The upload route keeps authentication, its rate buckets, multipart parsing and the response shape; the source-key answer, size and type checks, pre-link narrowing, content-hash dedup with alias memory, the quota-gated insert, the audit row and the background jobs move to src/lib/documents/ingest.ts unchanged, so a second way in can store through the same rules. --- src/app/api/documents/inbound/route.ts | 336 ++++------------------ src/lib/documents/ingest.ts | 378 +++++++++++++++++++++++++ 2 files changed, 435 insertions(+), 279 deletions(-) create mode 100644 src/lib/documents/ingest.ts diff --git a/src/app/api/documents/inbound/route.ts b/src/app/api/documents/inbound/route.ts index 70097db7b..f2f08dd2d 100644 --- a/src/app/api/documents/inbound/route.ts +++ b/src/app/api/documents/inbound/route.ts @@ -32,7 +32,6 @@ * instruction inside it. Storing it does nothing with its contents. */ import { Buffer } from "node:buffer"; -import { createHash } from "node:crypto"; import { NextResponse } from "next/server"; @@ -50,43 +49,38 @@ import { getClientIp, sanitiseZodIssues, } from "@/lib/api-response"; -import { getAiCapability } from "@/lib/ai/capabilities/gate"; -import { auditLog } from "@/lib/auth/audit"; import { prisma } from "@/lib/db"; import { hashQueryTokens } from "@/lib/documents/content-index"; +import { + ingestDocument, + isoDateToUtc, + personalUploadBucket, + UPLOAD_LIMIT_PER_HOUR, + UPLOAD_WINDOW_MS, +} from "@/lib/documents/ingest"; import { loadConditionLinks, loadDocumentEncounterLinks, - narrowOwnedEncounterIds, - narrowOwnedEpisodeIds, } from "@/lib/documents/links"; import { - encryptDocumentContent, serialiseDocument, type SerialisableDocument, } from "@/lib/documents/store"; -import { enqueueDocumentIndex } from "@/lib/jobs/document-index"; -import { enqueueDocumentSummary } from "@/lib/jobs/document-summary"; -import { enqueueDocumentThumbnail } from "@/lib/jobs/document-thumbnail"; import { DOCUMENTS_WRITE_SCOPE } from "@/lib/documents/scopes"; import { checkSourceLookupRateLimit, findSourceKey, - rememberSourceAlias, type SourceKeyMatch, } from "@/lib/documents/source-key"; import { acquireDocumentUploadSlot, - detectDocumentType, resolveDocumentLimits, resolveDocumentUploadLimitPerHour, } from "@/lib/documents/upload-policy"; import { withIdempotency } from "@/lib/idempotency"; import { BodyTooLargeError, readBoundedBody } from "@/lib/labs/ocr-upload"; -import { linkTargets } from "@/lib/links"; import { annotate } from "@/lib/logging/context"; import { requireModuleEnabled } from "@/lib/modules/gate"; -import { isP2002 } from "@/lib/prisma-errors"; import { checkRateLimit, rateLimitHeaders } from "@/lib/rate-limit"; import { documentCreateSchema, @@ -99,13 +93,6 @@ import type { Prisma } from "@/generated/prisma/client"; export const dynamic = "force-dynamic"; -/** - * A store-only upload touches no provider — the only abuse vector is disk, so - * a generous per-user ceiling is enough (extraction has its own tight gate). - */ -const UPLOAD_LIMIT_PER_HOUR = 60; -const UPLOAD_WINDOW_MS = 60 * 60 * 1000; - /** * Multipart envelope allowance on top of the per-file cap: boundaries plus * the small metadata fields (title / kind / documentDate / episodeIds). The @@ -114,11 +101,6 @@ const UPLOAD_WINDOW_MS = 60 * 60 * 1000; const MULTIPART_OVERHEAD_BYTES = 64 * 1024; const UPLOAD_BODY_TIMEOUT_MS = 30_000; -/** Parse a YYYY-MM-DD form field into a UTC midnight Date. */ -function isoDateToUtc(value: string): Date { - return new Date(`${value}T00:00:00.000Z`); -} - /** §3.2 — 413 fileTooLarge with the configured limit in `meta`. */ function fileTooLarge(maxFileBytes: number): NextResponse { return apiError("File is too large.", 413, { @@ -128,14 +110,6 @@ function fileTooLarge(maxFileBytes: number): NextResponse { }); } -/** Internal signal: the quota gate inside the insert transaction tripped. */ -class QuotaExceededError extends Error { - constructor(public readonly usedBytes: number) { - super("Document quota exceeded"); - this.name = "QuotaExceededError"; - } -} - /** What a single upload asked for, carried into the annotations. */ interface UploadContext { userId: string; @@ -357,7 +331,7 @@ async function processUpload( // draw on. The cookie / wildcard bucket is unchanged. const bucketKey = scoped ? `documents-upload:token:${auth.session.id}` - : `documents-upload:${user.id}`; + : personalUploadBucket(user.id); const rl = await checkRateLimit( bucketKey, scoped ? resolveDocumentUploadLimitPerHour() : UPLOAD_LIMIT_PER_HOUR, @@ -486,157 +460,55 @@ async function processUpload( } catch { return apiError("Failed to read uploaded file", 400); } - if (buffer.byteLength > limits.maxFileBytes) { - return fileTooLarge(limits.maxFileBytes); - } - if (buffer.byteLength === 0) { - return apiError("Uploaded file is empty", 422, { - errorCode: "documents.inbound.invalidMetadata", - }); - } - // §3.1 — magic-byte classification; the wire Content-Type is never trusted. - const detected = detectDocumentType( - buffer, - typeof file.name === "string" ? file.name : null, - ); - if (!detected) { - return apiError("This file type is not supported.", 415, { - errorCode: "documents.inbound.fileType", - reason: "unsupportedType", - }); - } - - // Pre-linking: every episode id must be a LIVE episode of the caller. - const episodeIds = await narrowOwnedEpisodeIds( - user.id, - parsed.data.episodeIds ?? [], - ); - if (episodeIds === null) { - return apiError("Episode not found", 404, { - errorCode: "documents.inbound.episodeNotFound", - }); - } - - // Same for the visit ids the review step offered. A refusal rather than a - // silent drop: the person saw a visit named and would otherwise watch the - // upload succeed without the link they asked for. - const encounterIds = await narrowOwnedEncounterIds( - user.id, - parsed.data.encounterIds ?? [], - ); - if (encounterIds === null) { - return apiError("Visit not found", 404, { - errorCode: "documents.inbound.encounterNotFound", - }); - } - - // sha256 of the PLAINTEXT for same-user duplicate detection. - const contentSha256 = createHash("sha256").update(buffer).digest("hex"); - - // Fast-path dedupe check; the partial unique index closes the race below. - const existing = await prisma.inboundDocument.findFirst({ - where: { userId: user.id, contentSha256, deletedAt: null }, - omit: { contentEncrypted: true }, + // Everything from here to the stored row is the shared ingest path + // (`src/lib/documents/ingest.ts`), the one the document picker's import + // stores through as well. + const result = await ingestDocument({ + userId: user.id, + scoped, + ipAddress: getClientIp(request), + bytes: buffer, + filename: typeof file.name === "string" ? file.name : null, + title: parsed.data.title ?? null, + kind: parsed.data.kind ?? null, + documentDate: parsed.data.documentDate ?? null, + episodeIds: parsed.data.episodeIds ?? [], + encounterIds: parsed.data.encounterIds ?? [], + sourceSystem, + sourceId, + aiDeferred: ctx.aiDeferred, + sourceKeyChecked: true, + limits, }); - if (existing) { - // An import sending bytes that are already stored under another key (or - // none) is answered with that document — and the key is remembered for - // it, so once the person deletes the document this key stays deleted too. - if (sourceSystem && sourceId) { - const remembered = await rememberSourceAlias( - user.id, - existing.id, - sourceSystem, - sourceId, - ); - if (remembered === "limit") return aliasLimitResponse(); - } - return duplicateResponse(ctx, existing); - } - // Quota gate + insert + pre-links in ONE transaction. Usage counts every - // non-purged row — tombstones still hold TOAST bytes, so "deleted" bytes - // are never invisible weight (undo-delete never changes usage). - let document: SerialisableDocument; - try { - document = await prisma.$transaction(async (tx) => { - // Serialise the quota gate per user: without this, N concurrent - // uploads all read the same SUM before any of them commits and the - // quota can be overshot by up to N × cap in one burst. The advisory - // lock is transaction-scoped (released on commit/rollback) and keyed - // on the user id, so uploads by different users never queue on each - // other. - // (`pg_advisory_xact_lock` returns void, which the client cannot - // deserialize as a column — selecting FROM it yields a plain int row.) - await tx.$queryRaw` - SELECT 1 AS locked - FROM pg_advisory_xact_lock(hashtextextended('documents-quota:' || ${user.id}, 0)) - `; - const rows = await tx.$queryRaw>` - SELECT COALESCE(SUM(byte_size), 0)::bigint AS used - FROM inbound_documents - WHERE user_id = ${user.id} - `; - const usedBytes = Number(rows[0]?.used ?? 0); - if (usedBytes + buffer.byteLength > limits.quotaBytes) { - throw new QuotaExceededError(usedBytes); - } - - const { content, codec } = encryptDocumentContent(buffer); - - // No mass assignment — every column is set field-by-field; `userId` - // comes from the session, never the body. `documentDate` defaults to - // the upload day so display == sort == filter (user-editable later). - const created = await tx.inboundDocument.create({ - data: { - userId: user.id, - kind: parsed.data.kind ?? "OTHER", - // Stored plaintext on purpose (mirrors `filename`) so the list can - // ILIKE-search + ORDER BY it. It MAY hold PHI the user types; that - // is the accepted tradeoff for server-side search/sort — the - // document body stays encrypted. - title: parsed.data.title ?? null, - filename: - typeof file.name === "string" ? file.name.slice(0, 255) : null, - mimeType: detected.mimeType, - byteSize: buffer.byteLength, - contentEncrypted: content, - contentCodec: codec, - contentSha256, - status: "STORED", - documentDate: parsed.data.documentDate - ? isoDateToUtc(parsed.data.documentDate) - : new Date(), - sourceSystem, - sourceId, - aiReadDeferred: ctx.aiDeferred, - }, - omit: { contentEncrypted: true }, + switch (result.kind) { + case "sourceKey": + return answerSourceKey(ctx, result.match); + case "tooLarge": + return fileTooLarge(result.maxFileBytes); + case "empty": + return apiError("Uploaded file is empty", 422, { + errorCode: "documents.inbound.invalidMetadata", }); - - if (episodeIds.length > 0) { - await linkTargets(tx, { - userId: user.id, - sourceKind: "document", - sourceId: created.id, - targetKind: "conditionEpisode", - targetIds: episodeIds, - }); - } - if (encounterIds.length > 0) { - await linkTargets(tx, { - userId: user.id, - sourceKind: "document", - sourceId: created.id, - targetKind: "encounter", - targetIds: encounterIds, - }); - } - return created; - }); - } catch (err) { - if (err instanceof QuotaExceededError) { + case "unsupportedType": + return apiError("This file type is not supported.", 415, { + errorCode: "documents.inbound.fileType", + reason: "unsupportedType", + }); + case "episodeNotFound": + return apiError("Episode not found", 404, { + errorCode: "documents.inbound.episodeNotFound", + }); + case "encounterNotFound": + return apiError("Visit not found", 404, { + errorCode: "documents.inbound.encounterNotFound", + }); + case "aliasLimit": + return aliasLimitResponse(); + case "duplicate": + return duplicateResponse(ctx, result.document); + case "quotaExceeded": // The figures are the owner's; a write-only token learns only that the // vault is full. return apiError("Storage quota exceeded.", 413, { @@ -644,107 +516,13 @@ async function processUpload( reason: "quotaExceeded", ...(scoped ? {} - : { quotaBytes: limits.quotaBytes, usedBytes: err.usedBytes }), + : { quotaBytes: result.quotaBytes, usedBytes: result.usedBytes }), }); - } - if (isP2002(err)) { - // A racing upload won one of the two partial unique indexes — the same - // source key, or the same bytes. Surface the winner exactly as the fast - // paths above would have. - if (sourceSystem && sourceId) { - const match = await findSourceKey(user.id, sourceSystem, sourceId); - if (match) return answerSourceKey(ctx, match); - } - const winner = await prisma.inboundDocument.findFirst({ - where: { userId: user.id, contentSha256, deletedAt: null }, - omit: { contentEncrypted: true }, - }); - if (winner) { - if (sourceSystem && sourceId) { - const remembered = await rememberSourceAlias( - user.id, - winner.id, - sourceSystem, - sourceId, - ); - if (remembered === "limit") return aliasLimitResponse(); - } - return duplicateResponse(ctx, winner); - } - } - throw err; - } - - await auditLog("documents.inbound.store", { - userId: user.id, - ipAddress: getClientIp(request), - details: { - documentId: document.id, - mime: detected.mimeType, - ...(scoped ? { scoped: true } : {}), - ...(sourceSystem ? { sourceSystem } : {}), - }, - }); - - annotate({ - action: { name: "documents.vault.upload" }, - meta: { - documentId: document.id, - byteSize: document.byteSize, - servingClass: detected.servingClass, - linked: episodeIds.length, - linkedVisits: encounterIds.length, - scoped, - sourceSystem, - aiDeferred: ctx.aiDeferred, - }, - }); - - // Auto-index the freshly stored document for content search: enqueue a - // fire-and-forget background job (provider-first, local text-layer fallback). - // The upload response never blocks on or fails because of indexing — the - // enqueue is not awaited and swallows its own errors (a missing boss or a - // transient send failure is a silent no-op). Only fresh inserts enqueue — a - // duplicate upload returns early above and never reaches here. - // - // `aiRead=defer` narrows it to the local text layer: an import of a whole - // archive must not turn into one provider call per file. The person reads - // them later, deliberately, from the document itself. - if (ctx.aiDeferred) { - void enqueueDocumentIndex(user.id, document.id, { localOnly: true }); - } else { - void enqueueDocumentIndex(user.id, document.id); - } - - // Render a preview thumbnail in the background too (pure local compute — no - // egress). Same fire-and-forget contract: the upload never blocks on or fails - // because of it, and a dropped enqueue is recoverable via the boot backfill. - // Only fresh inserts reach here (a duplicate returns early above). - void enqueueDocumentThumbnail(user.id, document.id); - - // Summarise the freshly stored document in the background — but ONLY when the - // `documentsAutoAiRead` opt-in is ON (the job re-checks it and the egress - // consent, and no-ops otherwise) and the `documentAi` capability is open for - // this record. The upload itself never depends on AI: the document is stored - // above whatever the capability says, and the index job still runs its - // provider-free text-layer path. The persisted summary shows on the detail - // view. Same fire-and-forget contract: the upload never blocks on or fails - // because of it. Only fresh inserts reach here (a duplicate returns early). - // - // A deferred upload skips it outright and keeps `summaryState: NONE`, so - // the detail sheet offers "Generate summary" rather than a pending state. - const documentAi = ctx.aiDeferred - ? { available: false as const, reason: "deferred" } - : await getAiCapability("documentAi"); - if (documentAi.available) { - void enqueueDocumentSummary(user.id, document.id); - } else { - annotate({ - action: { name: "documents.summary.enqueueSkipped" }, - meta: { documentId: document.id, reason: documentAi.reason }, - }); + case "stored": + break; } + const document = result.document; if (scoped) return receiptResponse(document.id, 201, { duplicate: false }); const [links, visitLinks] = await Promise.all([ diff --git a/src/lib/documents/ingest.ts b/src/lib/documents/ingest.ts new file mode 100644 index 000000000..06614a5c9 --- /dev/null +++ b/src/lib/documents/ingest.ts @@ -0,0 +1,378 @@ +/** + * One way into the vault for a file that has already been read into memory. + * + * The upload route (`POST /api/documents/inbound`, cookie, `["*"]` or a + * `documents:write` token) and the document picker's import route + * (`POST /api/documents/sources/{system}/import`, which fetches a picked + * document from Paperless-ngx or Papra) both store through here, so the rules a + * stored document is held to cannot differ by the way it arrived: + * + * - a source key answers before the bytes are looked at: a key already held + * (live, tombstoned, remembered as an alias or kept in the purge ledger) is + * a duplicate or a "deleted", never a second copy; + * - the per-file cap, the empty-file refusal and magic-byte classification + * (the wire Content-Type is never trusted); + * - pre-links to the caller's own live episodes and visits, refused when an + * id names nothing the caller owns; + * - sha256 duplicate detection against live rows, remembering the source key + * for the existing document so a later delete keeps that key deleted too; + * - the quota gate and the insert in one transaction under a per-user + * advisory lock, and the race on either partial unique index resolved the + * way the fast paths above would have answered; + * - the audit row, the wide-event annotation, and the background index, + * thumbnail and summary jobs (the summary only when the `documentAi` + * capability is open and the upload did not ask to defer AI reading). + * + * The callers own authentication, rate buckets, request parsing and the shape + * of their responses. This module returns what happened; it builds no HTTP. + */ +import type { Buffer } from "node:buffer"; +import { createHash } from "node:crypto"; + +import { getAiCapability } from "@/lib/ai/capabilities/gate"; +import { auditLog } from "@/lib/auth/audit"; +import { prisma } from "@/lib/db"; +import { + narrowOwnedEncounterIds, + narrowOwnedEpisodeIds, +} from "@/lib/documents/links"; +import { + findSourceKey, + rememberSourceAlias, + type SourceKeyMatch, +} from "@/lib/documents/source-key"; +import { + encryptDocumentContent, + type SerialisableDocument, +} from "@/lib/documents/store"; +import { detectDocumentType } from "@/lib/documents/upload-policy"; +import { enqueueDocumentIndex } from "@/lib/jobs/document-index"; +import { enqueueDocumentSummary } from "@/lib/jobs/document-summary"; +import { enqueueDocumentThumbnail } from "@/lib/jobs/document-thumbnail"; +import { linkTargets } from "@/lib/links"; +import { annotate } from "@/lib/logging/context"; +import { isP2002 } from "@/lib/prisma-errors"; +import type { + DocumentSourceSystemValue, + InboundDocumentKindValue, +} from "@/lib/validations/inbound-documents"; + +/** + * The person's own uploads per hour, from the web or the phone, and the + * picker's imports with them. A store-only upload touches no provider — the + * only abuse vector is disk, so a generous ceiling is enough. A narrow + * `documents:write` token draws on its own bucket instead + * (`resolveDocumentUploadLimitPerHour`). + */ +export const UPLOAD_LIMIT_PER_HOUR = 60; +export const UPLOAD_WINDOW_MS = 60 * 60 * 1000; + +/** The bucket a person's own uploads and picker imports share. */ +export function personalUploadBucket(userId: string): string { + return `documents-upload:${userId}`; +} + +/** Parse a YYYY-MM-DD value into a UTC midnight Date. */ +export function isoDateToUtc(value: string): Date { + return new Date(`${value}T00:00:00.000Z`); +} + +export interface IngestInput { + userId: string; + /** A narrow `documents:write` token rather than a session or `["*"]`. */ + scoped: boolean; + /** For the audit row. */ + ipAddress: string | null; + bytes: Buffer; + filename: string | null; + title: string | null; + kind: InboundDocumentKindValue | null; + /** YYYY-MM-DD; the upload day when absent. */ + documentDate: string | null; + episodeIds: string[]; + encounterIds: string[]; + sourceSystem: DocumentSourceSystemValue | null; + sourceId: string | null; + /** Hold back automatic AI reading (`aiRead=defer`). */ + aiDeferred: boolean; + /** + * The caller already asked about this source key (the upload's query-string + * form answers before the bucket is charged). Skips the second lookup. + */ + sourceKeyChecked: boolean; + /** The per-file cap and the quota, resolved by the caller. */ + limits: { maxFileBytes: number; quotaBytes: number }; +} + +export type IngestResult = + | { kind: "sourceKey"; match: SourceKeyMatch } + | { kind: "tooLarge"; maxFileBytes: number } + | { kind: "empty" } + | { kind: "unsupportedType" } + | { kind: "episodeNotFound" } + | { kind: "encounterNotFound" } + | { kind: "aliasLimit" } + | { kind: "duplicate"; document: SerialisableDocument } + | { kind: "quotaExceeded"; usedBytes: number; quotaBytes: number } + | { + kind: "stored"; + document: SerialisableDocument; + servingClass: string; + linkedEpisodes: number; + linkedVisits: number; + }; + +/** Internal signal: the quota gate inside the insert transaction tripped. */ +class QuotaExceededError extends Error { + constructor(public readonly usedBytes: number) { + super("Document quota exceeded"); + this.name = "QuotaExceededError"; + } +} + +/** A live row with the same bytes, remembering the source key for it. */ +async function answerDuplicate( + input: IngestInput, + existing: SerialisableDocument, +): Promise { + // An import sending bytes that are already stored under another key (or + // none) is answered with that document — and the key is remembered for it, + // so once the person deletes the document this key stays deleted too. + if (input.sourceSystem && input.sourceId) { + const remembered = await rememberSourceAlias( + input.userId, + existing.id, + input.sourceSystem, + input.sourceId, + ); + if (remembered === "limit") return { kind: "aliasLimit" }; + } + return { kind: "duplicate", document: existing }; +} + +export async function ingestDocument( + input: IngestInput, +): Promise { + const { userId, sourceSystem, sourceId, limits, bytes } = input; + + // A source key answers before the bytes are looked at: an import re-sending + // what it sent before is a duplicate, and one re-sending a document the + // person deleted is refused a second copy — live and tombstoned rows both + // count (the unique index has no `deleted_at` predicate), and past the purge + // the ledger remembers. + if (sourceSystem && sourceId && !input.sourceKeyChecked) { + const match = await findSourceKey(userId, sourceSystem, sourceId); + if (match) return { kind: "sourceKey", match }; + } + + if (bytes.byteLength > limits.maxFileBytes) { + return { kind: "tooLarge", maxFileBytes: limits.maxFileBytes }; + } + if (bytes.byteLength === 0) return { kind: "empty" }; + + // Magic-byte classification; the wire Content-Type is never trusted. + const detected = detectDocumentType(bytes, input.filename); + if (!detected) return { kind: "unsupportedType" }; + + // Pre-linking: every episode id must be a LIVE episode of the caller. + const episodeIds = await narrowOwnedEpisodeIds(userId, input.episodeIds); + if (episodeIds === null) return { kind: "episodeNotFound" }; + + // Same for the visit ids the review step offered. A refusal rather than a + // silent drop: the person saw a visit named and would otherwise watch the + // upload succeed without the link they asked for. + const encounterIds = await narrowOwnedEncounterIds( + userId, + input.encounterIds, + ); + if (encounterIds === null) return { kind: "encounterNotFound" }; + + // sha256 of the PLAINTEXT for same-user duplicate detection. + const contentSha256 = createHash("sha256").update(bytes).digest("hex"); + + // Fast-path dedupe check; the partial unique index closes the race below. + const existing = await prisma.inboundDocument.findFirst({ + where: { userId, contentSha256, deletedAt: null }, + omit: { contentEncrypted: true }, + }); + if (existing) return answerDuplicate(input, existing); + + // Quota gate + insert + pre-links in ONE transaction. Usage counts every + // non-purged row — tombstones still hold TOAST bytes, so "deleted" bytes + // are never invisible weight (undo-delete never changes usage). + let document: SerialisableDocument; + try { + document = await prisma.$transaction(async (tx) => { + // Serialise the quota gate per user: without this, N concurrent + // uploads all read the same SUM before any of them commits and the + // quota can be overshot by up to N × cap in one burst. The advisory + // lock is transaction-scoped (released on commit/rollback) and keyed + // on the user id, so uploads by different users never queue on each + // other. + // (`pg_advisory_xact_lock` returns void, which the client cannot + // deserialize as a column — selecting FROM it yields a plain int row.) + await tx.$queryRaw` + SELECT 1 AS locked + FROM pg_advisory_xact_lock(hashtextextended('documents-quota:' || ${userId}, 0)) + `; + const rows = await tx.$queryRaw>` + SELECT COALESCE(SUM(byte_size), 0)::bigint AS used + FROM inbound_documents + WHERE user_id = ${userId} + `; + const usedBytes = Number(rows[0]?.used ?? 0); + if (usedBytes + bytes.byteLength > limits.quotaBytes) { + throw new QuotaExceededError(usedBytes); + } + + const { content, codec } = encryptDocumentContent(bytes); + + // No mass assignment — every column is set field-by-field; `userId` + // comes from the session, never the body. `documentDate` defaults to + // the upload day so display == sort == filter (user-editable later). + const created = await tx.inboundDocument.create({ + data: { + userId, + kind: input.kind ?? "OTHER", + // Stored plaintext on purpose (mirrors `filename`) so the list can + // ILIKE-search + ORDER BY it. It MAY hold PHI the user types; that + // is the accepted tradeoff for server-side search/sort — the + // document body stays encrypted. + title: input.title, + filename: input.filename ? input.filename.slice(0, 255) : null, + mimeType: detected.mimeType, + byteSize: bytes.byteLength, + contentEncrypted: content, + contentCodec: codec, + contentSha256, + status: "STORED", + documentDate: input.documentDate + ? isoDateToUtc(input.documentDate) + : new Date(), + sourceSystem, + sourceId, + aiReadDeferred: input.aiDeferred, + }, + omit: { contentEncrypted: true }, + }); + + if (episodeIds.length > 0) { + await linkTargets(tx, { + userId, + sourceKind: "document", + sourceId: created.id, + targetKind: "conditionEpisode", + targetIds: episodeIds, + }); + } + if (encounterIds.length > 0) { + await linkTargets(tx, { + userId, + sourceKind: "document", + sourceId: created.id, + targetKind: "encounter", + targetIds: encounterIds, + }); + } + return created; + }); + } catch (err) { + if (err instanceof QuotaExceededError) { + return { + kind: "quotaExceeded", + usedBytes: err.usedBytes, + quotaBytes: limits.quotaBytes, + }; + } + if (isP2002(err)) { + // A racing upload won one of the two partial unique indexes — the same + // source key, or the same bytes. Surface the winner exactly as the fast + // paths above would have. + if (sourceSystem && sourceId) { + const match = await findSourceKey(userId, sourceSystem, sourceId); + if (match) return { kind: "sourceKey", match }; + } + const winner = await prisma.inboundDocument.findFirst({ + where: { userId, contentSha256, deletedAt: null }, + omit: { contentEncrypted: true }, + }); + if (winner) return answerDuplicate(input, winner); + } + throw err; + } + + await auditLog("documents.inbound.store", { + userId, + ipAddress: input.ipAddress, + details: { + documentId: document.id, + mime: detected.mimeType, + ...(input.scoped ? { scoped: true } : {}), + ...(sourceSystem ? { sourceSystem } : {}), + }, + }); + + annotate({ + action: { name: "documents.vault.upload" }, + meta: { + documentId: document.id, + byteSize: document.byteSize, + servingClass: detected.servingClass, + linked: episodeIds.length, + linkedVisits: encounterIds.length, + scoped: input.scoped, + sourceSystem, + aiDeferred: input.aiDeferred, + }, + }); + + // Auto-index the freshly stored document for content search: enqueue a + // fire-and-forget background job (provider-first, local text-layer fallback). + // The caller never blocks on or fails because of indexing — the enqueue is + // not awaited and swallows its own errors (a missing boss or a transient + // send failure is a silent no-op). Only fresh inserts enqueue — a duplicate + // returns early above and never reaches here. + // + // `aiRead=defer` narrows it to the local text layer: an import of a whole + // archive must not turn into one provider call per file. The person reads + // them later, deliberately, from the document itself. + if (input.aiDeferred) { + void enqueueDocumentIndex(userId, document.id, { localOnly: true }); + } else { + void enqueueDocumentIndex(userId, document.id); + } + + // Render a preview thumbnail in the background too (pure local compute — no + // egress). Same fire-and-forget contract; a dropped enqueue is recoverable + // via the boot backfill. + void enqueueDocumentThumbnail(userId, document.id); + + // Summarise the freshly stored document in the background — but ONLY when the + // `documentsAutoAiRead` opt-in is ON (the job re-checks it and the egress + // consent, and no-ops otherwise) and the `documentAi` capability is open for + // this record. Storing never depends on AI: the document is stored above + // whatever the capability says, and the index job still runs its + // provider-free text-layer path. + // + // A deferred upload skips it outright and keeps `summaryState: NONE`, so + // the detail sheet offers "Generate summary" rather than a pending state. + const documentAi = input.aiDeferred + ? { available: false as const, reason: "deferred" } + : await getAiCapability("documentAi"); + if (documentAi.available) { + void enqueueDocumentSummary(userId, document.id); + } else { + annotate({ + action: { name: "documents.summary.enqueueSkipped" }, + meta: { documentId: document.id, reason: documentAi.reason }, + }); + } + + return { + kind: "stored", + document, + servingClass: detected.servingClass, + linkedEpisodes: episodeIds.length, + linkedVisits: encounterIds.length, + }; +} From fdc60f69c2ab9f0ad0a74d35c759ca73dce49dc7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Sat, 26 Sep 2026 13:02:36 +0200 Subject: [PATCH 02/18] feat(documents): search and import from Paperless-ngx and Papra on request A person stores one connection per system (base address, Papra organization id, API token encrypted at rest) and can search it by document name with tag and date filters, then import picked documents one per request through the same ingest path as an upload: source-key answer before any download, content dedup, deleted stays deleted, quota, provenance, and the person's own AI reading setting. An import can link the document to the visit, condition or vaccination it was picked for. The operator lists the instances in DOCUMENT_SOURCE_ORIGINS; unset, the feature is off. Only listed origins are contacted, through safeFetch with the origin pinned, redirects refused, timeouts and byte caps. The routes are cookie-only and owner-only. The token is registered for rotation, removed by Delete all data, and not carried in backups. --- .env.example | 14 + .env.production.example | 14 + docker-compose.yml | 7 + docs/self-hosting/document-sources.md | 82 ++++ .../migration.sql | 22 + prisma/schema.prisma | 40 ++ scripts/env-manifest.json | 11 + scripts/rotate-encryption-key.ts | 10 + .../sources/[system]/import/route.ts | 294 ++++++++++++ .../api/documents/sources/[system]/route.ts | 191 ++++++++ .../sources/[system]/search/route.ts | 131 +++++ .../documents/sources/[system]/tags/route.ts | 52 ++ .../documents/sources/[system]/test/route.ts | 71 +++ src/app/api/documents/sources/route.ts | 54 +++ src/lib/crypto/encrypted-columns.ts | 8 + src/lib/data-wipe/wipe-plan.ts | 4 + src/lib/documents/source-key.ts | 61 +++ src/lib/documents/sources/clients.ts | 452 ++++++++++++++++++ src/lib/documents/sources/connections.ts | 117 +++++ src/lib/documents/sources/http.ts | 198 ++++++++ src/lib/documents/sources/origins.ts | 95 ++++ src/lib/documents/sources/route-support.ts | 156 ++++++ src/lib/documents/sources/types.ts | 127 +++++ src/lib/export/backup-plan.ts | 2 + src/lib/validations/document-sources.ts | 91 ++++ src/lib/validations/inbound-documents.ts | 2 +- 26 files changed, 2305 insertions(+), 1 deletion(-) create mode 100644 docs/self-hosting/document-sources.md create mode 100644 prisma/migrations/0356_document_source_connections/migration.sql create mode 100644 src/app/api/documents/sources/[system]/import/route.ts create mode 100644 src/app/api/documents/sources/[system]/route.ts create mode 100644 src/app/api/documents/sources/[system]/search/route.ts create mode 100644 src/app/api/documents/sources/[system]/tags/route.ts create mode 100644 src/app/api/documents/sources/[system]/test/route.ts create mode 100644 src/app/api/documents/sources/route.ts create mode 100644 src/lib/documents/sources/clients.ts create mode 100644 src/lib/documents/sources/connections.ts create mode 100644 src/lib/documents/sources/http.ts create mode 100644 src/lib/documents/sources/origins.ts create mode 100644 src/lib/documents/sources/route-support.ts create mode 100644 src/lib/documents/sources/types.ts create mode 100644 src/lib/validations/document-sources.ts diff --git a/.env.example b/.env.example index aa5a7e3c3..5474388c2 100644 --- a/.env.example +++ b/.env.example @@ -103,6 +103,20 @@ API_TOKEN_HMAC_KEY="REPLACE-WITH-OUTPUT-OF-openssl-rand-hex-32" # NOTIFICATION_PRIVATE_ORIGINS="" +# ----------------------------------------------------------------------------- +# Document picker: Paperless-ngx and Papra -- optional +# ----------------------------------------------------------------------------- +# The Paperless-ngx and Papra instances people may connect to pick documents +# from, comma-separated exact origins (scheme://host[:port], no path), e.g. +# "https://paperless.example.com,http://papra.lan:1221". Unset, the document +# picker is switched off. Only a listed origin is ever contacted, public or +# private; requests are pinned to it, redirects are not followed, and +# link-local, metadata and unspecified addresses stay refused even when +# listed. A malformed entry is logged once and grants nothing. +# See docs/self-hosting/document-sources.md. +# DOCUMENT_SOURCE_ORIGINS="" + + # ----------------------------------------------------------------------------- # AI Provider (optional) # ----------------------------------------------------------------------------- diff --git a/.env.production.example b/.env.production.example index ead149065..a297815e3 100644 --- a/.env.production.example +++ b/.env.production.example @@ -85,6 +85,20 @@ API_TOKEN_HMAC_KEY="" # NOTIFICATION_PRIVATE_ORIGINS="" +# ----------------------------------------------------------------------------- +# Document picker: Paperless-ngx and Papra -- optional +# ----------------------------------------------------------------------------- +# The Paperless-ngx and Papra instances people may connect to pick documents +# from, comma-separated exact origins (scheme://host[:port], no path), e.g. +# "https://paperless.example.com,http://papra.lan:1221". Unset, the document +# picker is switched off. Only a listed origin is ever contacted, public or +# private; requests are pinned to it, redirects are not followed, and +# link-local, metadata and unspecified addresses stay refused even when +# listed. A malformed entry is logged once and grants nothing. +# See docs/self-hosting/document-sources.md. +# DOCUMENT_SOURCE_ORIGINS="" + + # ----------------------------------------------------------------------------- # WHOOP -- optional integration # ----------------------------------------------------------------------------- diff --git a/docker-compose.yml b/docker-compose.yml index a24b90ba0..c595de613 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -215,6 +215,13 @@ x-healthlog-env: &healthlog_env # and unspecified addresses stay refused even when listed. See # docs/self-hosting/notifications.md. NOTIFICATION_PRIVATE_ORIGINS: "${NOTIFICATION_PRIVATE_ORIGINS:-}" + # Paperless-ngx and Papra instances the document picker may reach, + # comma-separated exact origins (e.g. "https://paperless.example.com, + # http://papra.lan:1221"). Unset, the picker is off. Only listed origins are + # ever contacted, public or private; redirects are not followed and + # link-local, metadata and unspecified addresses stay refused even when + # listed. See docs/self-hosting/document-sources.md. + DOCUMENT_SOURCE_ORIGINS: "${DOCUMENT_SOURCE_ORIGINS:-}" # Open-Meteo endpoints for the environment/weather correlation. Default to # the hosted service; point them at your own Open-Meteo deployment to keep # the coordinate lookups off a third party. diff --git a/docs/self-hosting/document-sources.md b/docs/self-hosting/document-sources.md new file mode 100644 index 000000000..96588771b --- /dev/null +++ b/docs/self-hosting/document-sources.md @@ -0,0 +1,82 @@ +# Document picker — Paperless-ngx and Papra + +People who keep their paperwork in Paperless-ngx or Papra can search it from +HealthLog and import the documents they pick: under "Link a document" on a +visit, a vaccination or a condition, and from the Documents page. They search +by document name, narrow by tag and date, tick what they want, and HealthLog +fetches those files and stores its own encrypted copy, exactly as if they had +been uploaded. + +The picker is off until the operator names the instances HealthLog may +contact. + +## Turning it on + +List the origins in the server environment: + +```env +DOCUMENT_SOURCE_ORIGINS="https://paperless.example.com,http://papra.lan:1221" +``` + +The variable is on the `docker-compose.yml` whitelist, so setting it in `.env` +is enough; restart the app container afterwards. Unset or empty, the settings +card and every import button stay hidden and the routes answer 404. + +Each entry is one `scheme://host[:port]`, the grammar +`NOTIFICATION_PRIVATE_ORIGINS` uses (see [notifications](notifications.md)): + +- only `http` and `https`; scheme, host and port must match what a person + enters (`https://paperless.example.com:8443` and + `https://paperless.example.com` are two different entries); +- no path, query, credentials, wildcard or CIDR; a person may still enter a + base address with a path (`https://example.com/paperless`) as long as its + origin is listed; +- the unspecified address, link-local and the cloud-metadata range can never + be listed, and a listed name that resolves there is refused when HealthLog + connects; +- a malformed entry is logged once at startup, reduced to scheme and host, and + grants nothing; the valid entries beside it keep working. + +Unlike the notification list, this list is the whole decision: an origin +that is not listed is refused even when it is a public address. Removing an +entry cuts off the connections that point at it on their next request; people +can still delete their stored connection. + +## What each person sets up + +Settings → Integrations → Document archives: + +- **Paperless-ngx:** the base address and an API token (Paperless-ngx: My + Profile → API token). Paperless-ngx 2.16 or later (API version 9). +- **Papra:** the base address, the organization id (from the Papra address + bar) and an API key with `documents:read` and `tags:read`. + +HealthLog tests the connection before saving it. The token is stored +encrypted (AES-256-GCM, rotated with the other keys by +`scripts/rotate-encryption-key.ts`) and is never shown again, not even in +part. + +## What leaves the server, and when + +Only requests the person starts: a search, the tag list for the filter, a +connection test, and one download per picked document. There is no background +sync and no polling. Every request goes to the listed origin only, with +redirects refused, a 10-second limit for searches and 60 seconds for a file, +and a size cap on every answer (1 MiB for a search page, the vault's per-file +limit for a document). + +## Limits and rules + +- Imports count against the person's normal upload allowance (60 an hour) and + storage quota. +- A document that is already in HealthLog is not stored twice, and one the + person deleted in HealthLog is not brought back by picking it again. These + are the same rules the document token and `scripts/import-documents.mjs` + follow. +- A picked document follows the person's own automatic AI reading setting. +- Only the owner of a record can connect an archive or import from it. People + with shared access to a record, whatever their level, do not see the picker. +- Connections are not part of backups. After a restore on another server, + each person reconnects (one form); the imported documents themselves are in + the backup with the rest of the vault. +- Delete all data and account deletion remove the connection and its token. diff --git a/prisma/migrations/0356_document_source_connections/migration.sql b/prisma/migrations/0356_document_source_connections/migration.sql new file mode 100644 index 000000000..6423236ed --- /dev/null +++ b/prisma/migrations/0356_document_source_connections/migration.sql @@ -0,0 +1,22 @@ +-- Document picker (#1038). One connection per person per system (Paperless-ngx +-- or Papra) with the API token encrypted at rest. The picker searches the +-- source and imports picked documents on request only; nothing polls. +CREATE TABLE "document_source_connections" ( + "id" TEXT NOT NULL, + "user_id" TEXT NOT NULL, + "system" VARCHAR(16) NOT NULL, + "base_url" VARCHAR(2048) NOT NULL, + "organization_id" VARCHAR(128), + "token_encrypted" TEXT NOT NULL, + "last_verified_at" TIMESTAMP(3), + "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" TIMESTAMP(3) NOT NULL, + CONSTRAINT "document_source_connections_pkey" PRIMARY KEY ("id") +); + +CREATE UNIQUE INDEX "document_source_connections_user_id_system_key" + ON "document_source_connections" ("user_id", "system"); + +ALTER TABLE "document_source_connections" + ADD CONSTRAINT "document_source_connections_user_id_fkey" + FOREIGN KEY ("user_id") REFERENCES "users"("id") ON DELETE CASCADE ON UPDATE CASCADE; diff --git a/prisma/schema.prisma b/prisma/schema.prisma index 7d725e8d3..47037b890 100644 --- a/prisma/schema.prisma +++ b/prisma/schema.prisma @@ -928,6 +928,9 @@ model User { // v1.39.2 — further source keys an import answered with an existing // document (same bytes under another key). See DocumentSourceAlias. documentSourceAliases DocumentSourceAlias[] + // v1.39.3 — the person's own connection to a Paperless-ngx or Papra + // instance for the document picker. See DocumentSourceConnection. + documentSourceConnections DocumentSourceConnection[] extractedFacts ExtractedFact[] documentConditionLinks DocumentConditionLink[] documentContentIndexes DocumentContentIndex[] @@ -8114,6 +8117,43 @@ model DocumentImportKey { @@map("document_import_keys") } +/// v1.39.3 (#1038) — one person's connection to a Paperless-ngx or Papra +/// instance, used by the document picker to search that system by name, tag +/// and date and import the documents the person picks. One per system per +/// person. Only reachable when the operator lists the instance's origin in +/// `DOCUMENT_SOURCE_ORIGINS`; the origin is re-checked on every request, so +/// removing it from the list cuts the connection off without touching the +/// row. No background job reads it: every request to the source is one the +/// person started. Not carried in backups (a third-party credential for a +/// service on the operator's network; see `NOT_IN_BACKUP_MODELS`); removed by +/// Delete all data and with the account. +model DocumentSourceConnection { + id String @id @default(cuid()) + userId String @map("user_id") + /// `PAPERLESS` | `PAPRA` (`DOCUMENT_PICKER_SYSTEMS` in + /// `src/lib/documents/sources/types.ts`). + system String @map("system") @db.VarChar(16) + /// Base address of the instance, origin plus an optional path. Plaintext: + /// its origin is compared with the operator's list on every use, and the + /// settings card shows it. Not health data. + baseUrl String @map("base_url") @db.VarChar(2048) + /// Papra organization id; null for Paperless-ngx. + organizationId String? @map("organization_id") @db.VarChar(128) + /// The API token (Paperless-ngx) or API key (Papra), AES-256-GCM via + /// `encrypt()`. No route returns it, in whole or in part. + tokenEncrypted String @map("token_encrypted") + /// Last time the source answered a test with this token: the save's live + /// check or the settings card's Test button. Shown on the card. + lastVerifiedAt DateTime? @map("last_verified_at") + createdAt DateTime @default(now()) @map("created_at") + updatedAt DateTime @updatedAt @map("updated_at") + + user User @relation(fields: [userId], references: [id], onDelete: Cascade) + + @@unique([userId, system]) + @@map("document_source_connections") +} + model ExtractedFact { id String @id @default(cuid()) documentId String @map("document_id") diff --git a/scripts/env-manifest.json b/scripts/env-manifest.json index 11fcfadb5..8992f4abc 100644 --- a/scripts/env-manifest.json +++ b/scripts/env-manifest.json @@ -245,6 +245,17 @@ } ] }, + { + "name": "Document picker", + "description": "Operator grant for the document picker, which lets a person search their own Paperless-ngx or Papra instance and import picked documents. Unset, the picker is off. Optional.", + "required": false, + "variables": [ + { + "name": "DOCUMENT_SOURCE_ORIGINS", + "purpose": "Comma-separated exact origins (scheme://host[:port], no path, no wildcard, no CIDR), e.g. \"https://paperless.example.com,http://papra.lan:1221\". Only listed origins are contacted, public or private; redirects are not followed; link-local, metadata and unspecified addresses stay refused even when listed; a malformed entry is logged once and grants nothing." + } + ] + }, { "name": "Off-host backups", "description": "Required ALL-OR-NONE: any of these without the rest disables backups silently.", diff --git a/scripts/rotate-encryption-key.ts b/scripts/rotate-encryption-key.ts index 0bac08bc9..20fa5124b 100644 --- a/scripts/rotate-encryption-key.ts +++ b/scripts/rotate-encryption-key.ts @@ -328,6 +328,16 @@ async function main() { ); } + // ───── Document picker (String) ───── + // Column: "tokenEncrypted" — a person's Paperless-ngx / Papra API token. + results.push( + await rotateStringColumn( + "DocumentSourceConnection", + "tokenEncrypted", + prisma.documentSourceConnection, + ), + ); + // ───── AppSettings — operator credentials (String) ───── // Columns: "adminAiKeyEncrypted" "webPushVapidPrivateKeyEncrypted" // "adminCodexAccessTokenEncrypted" "adminCodexRefreshTokenEncrypted" diff --git a/src/app/api/documents/sources/[system]/import/route.ts b/src/app/api/documents/sources/[system]/import/route.ts new file mode 100644 index 000000000..a2cad851e --- /dev/null +++ b/src/app/api/documents/sources/[system]/import/route.ts @@ -0,0 +1,294 @@ +/** + * Document picker (#1038): import one picked document from a connected + * archive into the vault. + * + * One document per request, so the sheet can show each one's progress and + * result, and no single request holds more than one file in memory. The path: + * + * 1. the link target, when the picker was opened from a record, must be one + * of the person's own live records (404 before anything leaves the + * server); + * 2. the source key answers first, without a download: a document already + * imported (or already holding this key as an alias) is a `duplicate`, + * and one the person deleted here is `deleted` — nothing is fetched and + * nothing is stored, exactly as the token upload answers the same key; + * 3. only then the vault's own upload allowance (60 an hour, shared with + * uploads from the web and the phone) and an upload slot are taken, the + * document's details and its original file are fetched (bounded by the + * vault's per-file cap), and the file goes through `ingestDocument`, the + * same function the upload route stores through: type check, content + * dedup, quota, provenance (`sourceSystem` / `sourceId`), the background + * jobs; + * 4. the document is linked to the record through `src/lib/links/`. + * + * AI reading follows the person's own setting, unlike the token upload's + * default of holding it back: the person picked this document by hand, as + * deliberately as an upload. The hourly allowance and the daily AI budget + * bound it either way. + * + * Cookie-only and owner-only; see `admitDocumentSourceCaller`. + */ +import { Buffer } from "node:buffer"; + +import { NextRequest, NextResponse } from "next/server"; + +import { apiHandler, requireAuth } from "@/lib/api-handler"; +import { + apiError, + apiValidationError, + getClientIp, + safeJson, + sanitiseZodIssues, +} from "@/lib/api-response"; +import { prisma } from "@/lib/db"; +import { + ingestDocument, + personalUploadBucket, + UPLOAD_LIMIT_PER_HOUR, + UPLOAD_WINDOW_MS, +} from "@/lib/documents/ingest"; +import { findSourceKey, type SourceKeyMatch } from "@/lib/documents/source-key"; +import { clientFor, loadConnection } from "@/lib/documents/sources/connections"; +import { + admitDocumentSourceCaller, + responseForSourceFailure, + sourceErrorResponse, + systemParam, +} from "@/lib/documents/sources/route-support"; +import type { + DocumentImportOutcome, + DocumentPickerLinkKind, + DocumentSourceImportDto, +} from "@/lib/documents/sources/types"; +import { + acquireDocumentUploadSlot, + resolveDocumentLimits, +} from "@/lib/documents/upload-policy"; +import { linkTargets } from "@/lib/links"; +import { annotate } from "@/lib/logging/context"; +import { checkRateLimit, rateLimitHeaders } from "@/lib/rate-limit"; +import { documentSourceImportSchema } from "@/lib/validations/document-sources"; + +export const dynamic = "force-dynamic"; + +type RouteParams = { params: Promise<{ system: string }> }; + +interface LinkTarget { + kind: DocumentPickerLinkKind; + id: string; +} + +/** Is the record the picker was opened from one of the person's live ones? */ +async function ownsLinkTarget( + userId: string, + link: LinkTarget, +): Promise { + const where = { id: link.id, userId, deletedAt: null }; + const select = { id: true } as const; + switch (link.kind) { + case "conditionEpisode": + return ( + (await prisma.illnessEpisode.findFirst({ where, select })) !== null + ); + case "encounter": + return (await prisma.encounter.findFirst({ where, select })) !== null; + case "vaccination": + return ( + (await prisma.vaccinationRecord.findFirst({ where, select })) !== null + ); + } +} + +/** File the document against the record, from the record's side. */ +async function linkDocument( + userId: string, + link: LinkTarget | undefined, + documentId: string, +): Promise { + if (!link) return false; + const result = await prisma.$transaction((tx) => + linkTargets(tx, { + userId, + sourceKind: link.kind, + sourceId: link.id, + targetKind: "document", + targetIds: [documentId], + }), + ); + return !result.unknownSource && result.unknownTargetIds.length === 0; +} + +function respond( + system: string, + outcome: DocumentImportOutcome, + documentId: string | null, + linked: boolean, +): NextResponse { + annotate({ + action: { name: "documents.sources.import" }, + meta: { system, outcome, linked, documentId }, + }); + const data: DocumentSourceImportDto = { outcome, documentId, linked }; + return NextResponse.json( + { data, error: null }, + { status: outcome === "imported" ? 201 : 200 }, + ); +} + +/** A key the vault already holds: link a live one, never revive a deleted one. */ +async function answerHeldKey( + system: string, + userId: string, + link: LinkTarget | undefined, + match: SourceKeyMatch, +): Promise { + if (match.state === "deleted") { + return respond(system, "deleted", match.id, false); + } + const linked = await linkDocument(userId, link, match.document.id); + return respond(system, "duplicate", match.document.id, linked); +} + +export const POST = apiHandler( + async (request: NextRequest, { params }: RouteParams) => { + const auth = await requireAuth(); + const refused = await admitDocumentSourceCaller(auth); + if (refused) return refused; + const system = await systemParam(params); + if (system instanceof Response) return system; + const userId = auth.user.id; + + const { data: body, error: jsonError } = await safeJson(request, { + maxBytes: 4 * 1024, + }); + if (jsonError) return jsonError; + const parsed = documentSourceImportSchema.safeParse(body); + if (!parsed.success) { + return apiValidationError( + "Invalid import request", + sanitiseZodIssues(parsed.error.issues), + 422, + ); + } + const { sourceId, kind, link } = parsed.data; + + const row = await loadConnection(userId, system); + if (!row) return sourceErrorResponse("notConnected"); + let client; + try { + client = clientFor(row); + } catch (err) { + return responseForSourceFailure(err); + } + + if (link && !(await ownsLinkTarget(userId, link))) { + return sourceErrorResponse("linkTargetNotFound"); + } + + // The source key answers before anything is downloaded or charged. + const held = await findSourceKey(userId, system, sourceId); + if (held) return answerHeldKey(system, userId, link, held); + + const rl = await checkRateLimit( + personalUploadBucket(userId), + UPLOAD_LIMIT_PER_HOUR, + UPLOAD_WINDOW_MS, + ); + if (!rl.allowed) { + return apiError("Too many uploads. Try again later.", 429, { + errorCode: "documents.inbound.rateLimited", + headers: rateLimitHeaders(rl), + }); + } + + const release = acquireDocumentUploadSlot(userId); + if (!release) { + return apiError( + "Too many uploads are already in progress. Try again shortly.", + 429, + { + errorCode: "documents.inbound.uploadBusy", + reason: "uploadBusy", + headers: { "Retry-After": "1" }, + }, + ); + } + + try { + const limits = await resolveDocumentLimits(userId); + let meta; + let file; + try { + meta = await client.document(sourceId); + file = await client.download(sourceId, limits.maxFileBytes); + } catch (err) { + return responseForSourceFailure(err); + } + + const result = await ingestDocument({ + userId, + scoped: false, + ipAddress: getClientIp(request), + bytes: Buffer.from(file.bytes), + filename: meta.filename ?? file.filename, + title: meta.title, + kind: kind ?? null, + documentDate: meta.date, + episodeIds: [], + encounterIds: [], + sourceSystem: system, + sourceId, + aiDeferred: false, + sourceKeyChecked: true, + limits, + }); + + switch (result.kind) { + case "sourceKey": + return answerHeldKey(system, userId, link, result.match); + case "duplicate": { + // The same bytes were already in the vault (an upload, or another + // archive); the key is now remembered for that document. + const linked = await linkDocument(userId, link, result.document.id); + return respond(system, "duplicate", result.document.id, linked); + } + case "stored": { + const linked = await linkDocument(userId, link, result.document.id); + return respond(system, "imported", result.document.id, linked); + } + case "tooLarge": + return apiError("File is too large.", 413, { + errorCode: "documents.inbound.fileTooLarge", + reason: "fileTooLarge", + maxFileBytes: result.maxFileBytes, + }); + case "empty": + case "unsupportedType": + return apiError("This file type is not supported.", 415, { + errorCode: "documents.inbound.fileType", + reason: "unsupportedType", + }); + case "quotaExceeded": + return apiError("Storage quota exceeded.", 413, { + errorCode: "documents.inbound.quotaExceeded", + reason: "quotaExceeded", + quotaBytes: result.quotaBytes, + usedBytes: result.usedBytes, + }); + case "aliasLimit": + return apiError( + "This file is already stored under too many source ids.", + 409, + { errorCode: "documents.inbound.sourceAliasLimit" }, + ); + case "episodeNotFound": + case "encounterNotFound": + // No pre-links are passed on this path; unreachable, but a result + // the switch does not name must not fall through silently. + return sourceErrorResponse("linkTargetNotFound"); + } + } finally { + release(); + } + }, +); diff --git a/src/app/api/documents/sources/[system]/route.ts b/src/app/api/documents/sources/[system]/route.ts new file mode 100644 index 000000000..0057e4c24 --- /dev/null +++ b/src/app/api/documents/sources/[system]/route.ts @@ -0,0 +1,191 @@ +/** + * Document picker (#1038): save or remove the person's connection to one + * document archive (`paperless` or `papra`). + * + * PUT validates the base address against the operator's list + * (`DOCUMENT_SOURCE_ORIGINS`), tests the connection live before storing + * anything (a wrong token or an unreachable host is reported here, not on the + * first search), then stores the token encrypted. A PUT without a token keeps + * the stored one, so the address or the Papra organization can change without + * typing the key again; the first save must carry it. The response is the + * connection without its token, like every read of it. + * + * DELETE removes the row, and with it the token. + * + * Cookie-only and owner-only; see `admitDocumentSourceCaller`. + */ +import { NextRequest } from "next/server"; + +import { apiHandler, requireAuth } from "@/lib/api-handler"; +import { + apiError, + apiSuccess, + apiValidationError, + getClientIp, + safeJson, + sanitiseZodIssues, +} from "@/lib/api-response"; +import { auditLog } from "@/lib/auth/audit"; +import { decrypt, encrypt } from "@/lib/crypto"; +import { prisma } from "@/lib/db"; +import { sourceClient } from "@/lib/documents/sources/clients"; +import { + loadConnection, + toConnectionDto, +} from "@/lib/documents/sources/connections"; +import { DocumentSourceError } from "@/lib/documents/sources/http"; +import { evaluateSourceBaseUrl } from "@/lib/documents/sources/origins"; +import { + admitDocumentSourceCaller, + checkConnectRateLimit, + responseForSourceFailure, + sourceErrorResponse, + systemParam, +} from "@/lib/documents/sources/route-support"; +import { annotate } from "@/lib/logging/context"; +import { rateLimitHeaders } from "@/lib/rate-limit"; +import { documentSourceSaveSchema } from "@/lib/validations/document-sources"; + +export const dynamic = "force-dynamic"; + +type RouteParams = { params: Promise<{ system: string }> }; + +export const PUT = apiHandler( + async (request: NextRequest, { params }: RouteParams) => { + const auth = await requireAuth(); + const refused = await admitDocumentSourceCaller(auth); + if (refused) return refused; + const system = await systemParam(params); + if (system instanceof Response) return system; + const userId = auth.user.id; + annotate({ action: { name: "documents.sources.save" }, meta: { system } }); + + const rl = await checkConnectRateLimit(userId); + if (!rl.allowed) { + return sourceErrorResponse( + "rateLimited", + undefined, + rateLimitHeaders(rl), + ); + } + + const { data: body, error: jsonError } = await safeJson(request, { + maxBytes: 8 * 1024, + }); + if (jsonError) return jsonError; + const parsed = documentSourceSaveSchema.safeParse(body); + if (!parsed.success) { + return apiValidationError( + "Invalid connection details", + sanitiseZodIssues(parsed.error.issues), + 422, + ); + } + + const verdict = evaluateSourceBaseUrl(parsed.data.baseUrl); + if (!verdict.ok) { + return verdict.reason === "notAllowed" + ? sourceErrorResponse("originNotAllowed") + : apiError("Enter the full address, starting with https://.", 422, { + errorCode: "documents.sources.invalidAddress", + }); + } + + const organizationId = + system === "PAPRA" ? (parsed.data.organizationId ?? null) : null; + if (system === "PAPRA" && !organizationId) { + return apiError("Papra needs the organization id.", 422, { + errorCode: "documents.sources.organizationRequired", + }); + } + + const existing = await loadConnection(userId, system); + const token = + parsed.data.token ?? (existing ? decrypt(existing.tokenEncrypted) : null); + if (!token) { + return apiError("Enter the API token.", 422, { + errorCode: "documents.sources.tokenRequired", + }); + } + + // Live check before anything is stored: the row only ever holds a + // connection that answered once. + try { + await sourceClient({ + system, + origin: verdict.origin, + baseUrl: verdict.baseUrl, + organizationId, + token, + }).test(); + } catch (err) { + if (err instanceof DocumentSourceError) { + annotate({ meta: { refused: err.code } }); + } + return responseForSourceFailure(err); + } + + const now = new Date(); + const tokenEncrypted = encrypt(token); + // Field by field; `userId` from the session, never the body. + const saved = await prisma.documentSourceConnection.upsert({ + where: { userId_system: { userId, system } }, + create: { + userId, + system, + baseUrl: verdict.baseUrl, + organizationId, + tokenEncrypted, + lastVerifiedAt: now, + }, + update: { + baseUrl: verdict.baseUrl, + organizationId, + tokenEncrypted, + lastVerifiedAt: now, + }, + select: { + system: true, + baseUrl: true, + organizationId: true, + lastVerifiedAt: true, + }, + }); + + await auditLog("documents.sources.connect", { + userId, + ipAddress: getClientIp(request), + details: { system, origin: verdict.origin, replaced: existing !== null }, + }); + + return apiSuccess(toConnectionDto(saved)); + }, +); + +export const DELETE = apiHandler( + async (request: NextRequest, { params }: RouteParams) => { + const auth = await requireAuth(); + // No list needed to forget a connection: an operator who removed the + // origin must not leave the person unable to delete the stored token. + const refused = await admitDocumentSourceCaller(auth, { needsList: false }); + if (refused) return refused; + const system = await systemParam(params); + if (system instanceof Response) return system; + annotate({ + action: { name: "documents.sources.disconnect" }, + meta: { system }, + }); + + const { count } = await prisma.documentSourceConnection.deleteMany({ + where: { userId: auth.user.id, system }, + }); + if (count > 0) { + await auditLog("documents.sources.disconnect", { + userId: auth.user.id, + ipAddress: getClientIp(request), + details: { system }, + }); + } + return apiSuccess({ disconnected: count > 0 }); + }, +); diff --git a/src/app/api/documents/sources/[system]/search/route.ts b/src/app/api/documents/sources/[system]/search/route.ts new file mode 100644 index 000000000..7296048e8 --- /dev/null +++ b/src/app/api/documents/sources/[system]/search/route.ts @@ -0,0 +1,131 @@ +/** + * Document picker (#1038): search one connected document archive. + * + * `q` searches the document's name (the title in Paperless-ngx, the name in + * Papra), `tag` narrows to one of the source's tags by its id, `from` / `to` + * to a date range (inclusive), `page` pages through 25 at a time. The answer + * is a minimal row per document (source id, title, date, tag names, size when + * the source says it) plus what HealthLog already holds under that source key: + * `new`, `imported` (with the vault document's id) or `deleted` (the person + * deleted it here; importing it again is refused). Nothing is stored. + * + * Sixty searches a minute, shared with the tag read. Cookie-only and + * owner-only; see `admitDocumentSourceCaller`. + */ +import { NextRequest } from "next/server"; + +import { apiHandler, requireAuth } from "@/lib/api-handler"; +import { + apiSuccess, + apiValidationError, + sanitiseZodIssues, +} from "@/lib/api-response"; +import { findSourceKeyStates } from "@/lib/documents/source-key"; +import type { SourceListItem } from "@/lib/documents/sources/clients"; +import { openConnection } from "@/lib/documents/sources/connections"; +import { + admitDocumentSourceCaller, + checkSearchRateLimit, + responseForSourceFailure, + sourceErrorResponse, + systemParam, +} from "@/lib/documents/sources/route-support"; +import type { + DocumentSourceResultDto, + DocumentSourceSearchDto, +} from "@/lib/documents/sources/types"; +import { annotate } from "@/lib/logging/context"; +import { rateLimitHeaders } from "@/lib/rate-limit"; +import { documentSourceSearchSchema } from "@/lib/validations/document-sources"; + +export const dynamic = "force-dynamic"; + +type RouteParams = { params: Promise<{ system: string }> }; + +export const GET = apiHandler( + async (request: NextRequest, { params }: RouteParams) => { + const auth = await requireAuth(); + const refused = await admitDocumentSourceCaller(auth); + if (refused) return refused; + const system = await systemParam(params); + if (system instanceof Response) return system; + const userId = auth.user.id; + + const url = new URL(request.url); + const parsed = documentSourceSearchSchema.safeParse({ + q: url.searchParams.get("q") ?? undefined, + tag: url.searchParams.get("tag") ?? undefined, + from: url.searchParams.get("from") ?? undefined, + to: url.searchParams.get("to") ?? undefined, + page: url.searchParams.get("page") ?? undefined, + }); + if (!parsed.success) { + return apiValidationError( + "Invalid search", + sanitiseZodIssues(parsed.error.issues), + 422, + ); + } + const { q, tag, from, to, page } = parsed.data; + + const rl = await checkSearchRateLimit(userId); + if (!rl.allowed) { + return sourceErrorResponse( + "rateLimited", + undefined, + rateLimitHeaders(rl), + ); + } + + let found: { items: SourceListItem[]; hasMore: boolean }; + try { + const client = await openConnection(userId, system); + found = await client.search({ + q, + tagId: tag ?? null, + from: from ?? null, + to: to ?? null, + page, + }); + } catch (err) { + return responseForSourceFailure(err); + } + + const states = await findSourceKeyStates( + userId, + system, + found.items.map((item) => item.sourceId), + ); + const results: DocumentSourceResultDto[] = found.items.map((item) => { + const held = states.get(item.sourceId); + return { + sourceId: item.sourceId, + title: item.title, + date: item.date, + tags: item.tags, + sizeBytes: item.sizeBytes, + state: held?.state ?? "new", + documentId: held?.state === "imported" ? held.documentId : null, + }; + }); + + // Counts only: the query and the titles are the person's own words and + // stay out of the wide event. + annotate({ + action: { name: "documents.sources.search" }, + meta: { + system, + page, + count: results.length, + filtered: Boolean(q || tag || from || to), + }, + }); + + const body: DocumentSourceSearchDto = { + results, + page, + hasMore: found.hasMore, + }; + return apiSuccess(body); + }, +); diff --git a/src/app/api/documents/sources/[system]/tags/route.ts b/src/app/api/documents/sources/[system]/tags/route.ts new file mode 100644 index 000000000..4b788608c --- /dev/null +++ b/src/app/api/documents/sources/[system]/tags/route.ts @@ -0,0 +1,52 @@ +/** + * Document picker (#1038): the tags of one connected document archive, for the + * picker's tag filter. The first 250 by name, as `{ id, name }` in the + * source's own ids. Shares the search allowance (60 a minute). + * + * Cookie-only and owner-only; see `admitDocumentSourceCaller`. + */ +import { NextRequest } from "next/server"; + +import { apiHandler, requireAuth } from "@/lib/api-handler"; +import { apiSuccess } from "@/lib/api-response"; +import { openConnection } from "@/lib/documents/sources/connections"; +import { + admitDocumentSourceCaller, + checkSearchRateLimit, + responseForSourceFailure, + sourceErrorResponse, + systemParam, +} from "@/lib/documents/sources/route-support"; +import { annotate } from "@/lib/logging/context"; +import { rateLimitHeaders } from "@/lib/rate-limit"; + +export const dynamic = "force-dynamic"; + +type RouteParams = { params: Promise<{ system: string }> }; + +export const GET = apiHandler( + async (_request: NextRequest, { params }: RouteParams) => { + const auth = await requireAuth(); + const refused = await admitDocumentSourceCaller(auth); + if (refused) return refused; + const system = await systemParam(params); + if (system instanceof Response) return system; + annotate({ action: { name: "documents.sources.tags" }, meta: { system } }); + + const rl = await checkSearchRateLimit(auth.user.id); + if (!rl.allowed) { + return sourceErrorResponse( + "rateLimited", + undefined, + rateLimitHeaders(rl), + ); + } + + try { + const client = await openConnection(auth.user.id, system); + return apiSuccess({ tags: await client.tags() }); + } catch (err) { + return responseForSourceFailure(err); + } + }, +); diff --git a/src/app/api/documents/sources/[system]/test/route.ts b/src/app/api/documents/sources/[system]/test/route.ts new file mode 100644 index 000000000..536b0c7d1 --- /dev/null +++ b/src/app/api/documents/sources/[system]/test/route.ts @@ -0,0 +1,71 @@ +/** + * Document picker (#1038): test the saved connection to one document archive. + * + * One request to the source with the stored token, answered `{ ok, latencyMs }` + * or with the reason it failed (`meta.errorCode`, and `meta.upstreamStatus` + * when the source answered at all; its body is never passed on). A success + * stamps `lastVerifiedAt`. Shares the save's allowance of ten a minute. + * + * Cookie-only and owner-only; see `admitDocumentSourceCaller`. + */ +import { NextRequest } from "next/server"; + +import { apiHandler, requireAuth } from "@/lib/api-handler"; +import { apiSuccess } from "@/lib/api-response"; +import { + markVerified, + openConnection, +} from "@/lib/documents/sources/connections"; +import { DocumentSourceError } from "@/lib/documents/sources/http"; +import { + admitDocumentSourceCaller, + checkConnectRateLimit, + responseForSourceFailure, + sourceErrorResponse, + systemParam, +} from "@/lib/documents/sources/route-support"; +import { annotate } from "@/lib/logging/context"; +import { rateLimitHeaders } from "@/lib/rate-limit"; + +export const dynamic = "force-dynamic"; + +type RouteParams = { params: Promise<{ system: string }> }; + +export const POST = apiHandler( + async (_request: NextRequest, { params }: RouteParams) => { + const auth = await requireAuth(); + const refused = await admitDocumentSourceCaller(auth); + if (refused) return refused; + const system = await systemParam(params); + if (system instanceof Response) return system; + const userId = auth.user.id; + annotate({ action: { name: "documents.sources.test" }, meta: { system } }); + + const rl = await checkConnectRateLimit(userId); + if (!rl.allowed) { + return sourceErrorResponse( + "rateLimited", + undefined, + rateLimitHeaders(rl), + ); + } + + const start = performance.now(); + try { + const client = await openConnection(userId, system); + await client.test(); + } catch (err) { + if (err instanceof DocumentSourceError) { + annotate({ meta: { refused: err.code } }); + } + return responseForSourceFailure(err); + } + const latencyMs = Math.round(performance.now() - start); + const verifiedAt = await markVerified(userId, system); + return apiSuccess({ + ok: true, + latencyMs, + lastVerifiedAt: verifiedAt.toISOString(), + }); + }, +); diff --git a/src/app/api/documents/sources/route.ts b/src/app/api/documents/sources/route.ts new file mode 100644 index 000000000..0a91b184b --- /dev/null +++ b/src/app/api/documents/sources/route.ts @@ -0,0 +1,54 @@ +/** + * Document picker (#1038): which document archives this person can pick from. + * + * `available` says whether the operator listed any origin in + * `DOCUMENT_SOURCE_ORIGINS`; `allowedOrigins` lists them so the settings card + * can say which addresses will be accepted; `connections` are the person's own + * saved connections, never with their token (`hasToken` is the literal true). + * No request leaves the server here. + * + * Cookie-only and owner-only like every route under this path + * (`admitDocumentSourceCaller`). With the list unset it answers 200 + * `available: false` rather than 404, because this is the one read the client + * makes to decide whether to show the picker at all. + */ +import { apiHandler, requireAuth } from "@/lib/api-handler"; +import { apiSuccess } from "@/lib/api-response"; +import { + listConnections, + toConnectionDto, +} from "@/lib/documents/sources/connections"; +import { + documentSourcesAvailable, + listedDocumentSourceOrigins, +} from "@/lib/documents/sources/origins"; +import type { DocumentSourcesStatusDto } from "@/lib/documents/sources/types"; +import { admitDocumentSourceCaller } from "@/lib/documents/sources/route-support"; +import { annotate } from "@/lib/logging/context"; + +export const dynamic = "force-dynamic"; + +export const GET = apiHandler(async () => { + const auth = await requireAuth(); + const refused = await admitDocumentSourceCaller(auth, { needsList: false }); + if (refused) return refused; + + annotate({ action: { name: "documents.sources.status" } }); + + if (!documentSourcesAvailable()) { + const off: DocumentSourcesStatusDto = { + available: false, + allowedOrigins: [], + connections: [], + }; + return apiSuccess(off); + } + + const rows = await listConnections(auth.user.id); + const status: DocumentSourcesStatusDto = { + available: true, + allowedOrigins: listedDocumentSourceOrigins(), + connections: rows.map(toConnectionDto), + }; + return apiSuccess(status); +}); diff --git a/src/lib/crypto/encrypted-columns.ts b/src/lib/crypto/encrypted-columns.ts index 684228dba..6759e5d7e 100644 --- a/src/lib/crypto/encrypted-columns.ts +++ b/src/lib/crypto/encrypted-columns.ts @@ -161,6 +161,14 @@ export const ENCRYPTED_COLUMNS: readonly EncryptedColumn[] = [ { model: "GoogleHealthConnection", field: "accessToken", kind: "string" }, { model: "GoogleHealthConnection", field: "refreshToken", kind: "string" }, + // ───── Document picker (v1.39.3, #1038) ───── + // A person's API token for their Paperless-ngx or Papra instance. + { + model: "DocumentSourceConnection", + field: "tokenEncrypted", + kind: "string", + }, + // ───── AppSettings — operator credentials ───── { model: "AppSettings", field: "adminAiKeyEncrypted", kind: "string" }, { diff --git a/src/lib/data-wipe/wipe-plan.ts b/src/lib/data-wipe/wipe-plan.ts index 3bb1bcf60..1e4f2ff9e 100644 --- a/src/lib/data-wipe/wipe-plan.ts +++ b/src/lib/data-wipe/wipe-plan.ts @@ -155,6 +155,10 @@ export const WIPE_MODELS = [ // Further import keys that resolved to a document by content; a child of // the document, removed before it so the counts are truthful. "DocumentSourceAlias", + // The person's own connections to Paperless-ngx / Papra (#1038), with their + // encrypted API tokens. A wipe that left them would leave a working + // credential to another system behind an account that erased everything. + "DocumentSourceConnection", "ExtractedFact", "ImportJob", diff --git a/src/lib/documents/source-key.ts b/src/lib/documents/source-key.ts index 0c5656f5b..61fa6580d 100644 --- a/src/lib/documents/source-key.ts +++ b/src/lib/documents/source-key.ts @@ -107,3 +107,64 @@ export function checkSourceLookupRateLimit( SOURCE_LOOKUP_WINDOW_MS, ); } + +/** What the vault holds for one source key, as the picker lists it. */ +export type SourceKeyState = + { state: "imported"; documentId: string } | { state: "deleted" }; + +/** + * {@link findSourceKey} for a page of results at once: the same three places, + * three queries for the whole page instead of three per result. A key held by + * a live document answers `imported` with its id; one held by a tombstone, or + * kept by the purge ledger, answers `deleted`. A key the vault does not know + * is absent from the map. + */ +export async function findSourceKeyStates( + userId: string, + sourceSystem: DocumentSourceSystemValue, + sourceIds: string[], +): Promise> { + const states = new Map(); + const ids = [...new Set(sourceIds)]; + if (ids.length === 0) return states; + + const [own, aliases, purged] = await Promise.all([ + prisma.inboundDocument.findMany({ + where: { userId, sourceSystem, sourceId: { in: ids } }, + select: { id: true, sourceId: true, deletedAt: true }, + }), + prisma.documentSourceAlias.findMany({ + where: { userId, sourceSystem, sourceId: { in: ids } }, + select: { + sourceId: true, + document: { select: { id: true, deletedAt: true } }, + }, + }), + prisma.documentImportKey.findMany({ + where: { userId, sourceSystem, sourceId: { in: ids } }, + select: { sourceId: true }, + }), + ]); + + // Lowest precedence first, so the document's own key wins, as it does in + // `findSourceKey`. + for (const row of purged) states.set(row.sourceId, { state: "deleted" }); + for (const row of aliases) { + states.set( + row.sourceId, + row.document.deletedAt + ? { state: "deleted" } + : { state: "imported", documentId: row.document.id }, + ); + } + for (const row of own) { + if (!row.sourceId) continue; + states.set( + row.sourceId, + row.deletedAt + ? { state: "deleted" } + : { state: "imported", documentId: row.id }, + ); + } + return states; +} diff --git a/src/lib/documents/sources/clients.ts b/src/lib/documents/sources/clients.ts new file mode 100644 index 000000000..7bca6ed92 --- /dev/null +++ b/src/lib/documents/sources/clients.ts @@ -0,0 +1,452 @@ +/** + * The two source adapters the document picker speaks: Paperless-ngx and Papra. + * + * Each turns the picker's four questions (does this connection work, which + * tags exist, which documents match, give me this one) into the source's own + * REST calls, and each answer into the picker's minimal shapes. Everything + * leaves through `./http.ts`, so the origin pin, the redirect refusal, the + * timeouts and the byte caps are not something an adapter can forget. + * + * Both adapters re-check tag and date filters on what comes back. A server + * that ignores a parameter (an older Paperless, a Papra whose search syntax + * changed) must not widen a result into documents the person filtered out; + * the import script makes the same check for the same reason. + * + * API versions: Paperless-ngx is asked for API version 9 (Paperless-ngx 2.16 + * and later, the version where `created` became a plain date; it answers 406 + * to a version it does not speak). Papra's endpoints are the documented + * organization-scoped ones (`/api/organizations/{id}/documents`, `/tags`, + * `/documents/{id}`, `/documents/{id}/file`). + */ +import { DOCUMENT_TITLE_MAX } from "@/lib/validations/inbound-documents"; + +import { + DocumentSourceError, + sourceDownload, + sourceJson, + type SourceTarget, +} from "./http"; +import { + DOCUMENT_PICKER_MAX_TAGS, + DOCUMENT_PICKER_PAGE_SIZE, + type DocumentPickerSystem, + type DocumentSourceTagDto, +} from "./types"; + +/** Paperless-ngx API version the picker speaks (Paperless-ngx 2.16+). */ +export const PAPERLESS_API_VERSION = 9; + +export interface SourceSearchParams { + /** Name to search for; empty lists everything. */ + q: string; + /** A tag id as the source names it. */ + tagId: string | null; + /** YYYY-MM-DD, inclusive. */ + from: string | null; + to: string | null; + /** 1-based. */ + page: number; +} + +export interface SourceListItem { + sourceId: string; + title: string; + date: string | null; + tags: string[]; + sizeBytes: number | null; +} + +export interface SourceDocumentMeta { + title: string; + date: string | null; + filename: string | null; +} + +export interface SourceClient { + readonly system: DocumentPickerSystem; + test(): Promise; + tags(): Promise; + search( + params: SourceSearchParams, + ): Promise<{ items: SourceListItem[]; hasMore: boolean }>; + document(sourceId: string): Promise; + download( + sourceId: string, + maxFileBytes: number, + ): Promise<{ bytes: Uint8Array; filename: string | null }>; +} + +export interface SourceConnection { + system: DocumentPickerSystem; + origin: string; + baseUrl: string; + organizationId: string | null; + token: string; +} + +// ── Defensive readers ────────────────────────────────────────────────────── + +type Json = Record; + +function asRecord(value: unknown): Json | null { + return value && typeof value === "object" && !Array.isArray(value) + ? (value as Json) + : null; +} + +function asArray(value: unknown): unknown[] { + return Array.isArray(value) ? value : []; +} + +function asText(value: unknown): string | null { + if (typeof value === "string") return value; + if (typeof value === "number" && Number.isFinite(value)) return String(value); + return null; +} + +/** The body is not the shape the API documents: most often a base address + * that points at the web app instead of the instance. */ +function shapeError(): DocumentSourceError { + return new DocumentSourceError("badResponse"); +} + +/** YYYY-MM-DD from an ISO date or date-time, or null. */ +export function dayOf(value: unknown): string | null { + if (typeof value !== "string") return null; + const day = value.slice(0, 10); + return /^\d{4}-\d{2}-\d{2}$/.test(day) ? day : null; +} + +function stripExtension(name: string | null): string { + return name ? name.replace(/\.[A-Za-z0-9]{1,5}$/, "") : ""; +} + +/** + * A title that is not blank and fits the vault: the source's title, else its + * file name without the extension, else a placeholder naming the source id. + */ +export function titleOf( + title: string | null, + filename: string | null, + fallback: string, +): string { + const picked = + (title ?? "").trim() || stripExtension(filename).trim() || fallback; + return picked.slice(0, DOCUMENT_TITLE_MAX); +} + +/** Is `day` inside the inclusive range? A document without a date is kept + * only when no range was asked for. */ +export function withinRange( + day: string | null, + from: string | null, + to: string | null, +): boolean { + if (!from && !to) return true; + if (!day) return false; + if (from && day < from) return false; + if (to && day > to) return false; + return true; +} + +// ── Paperless-ngx ────────────────────────────────────────────────────────── + +function paperless(connection: SourceConnection): SourceClient { + const target: SourceTarget = { + origin: connection.origin, + baseUrl: connection.baseUrl, + headers: { + Authorization: `Token ${connection.token}`, + Accept: `application/json; version=${PAPERLESS_API_VERSION}`, + }, + }; + const download: SourceTarget = { + ...target, + headers: { Authorization: `Token ${connection.token}` }, + }; + + /** A server that says which API version it speaks, and it is too old. */ + function checkVersion(response: Response): void { + const raw = response.headers.get("x-api-version"); + if (raw === null) return; + const version = Number(raw); + if (Number.isFinite(version) && version < PAPERLESS_API_VERSION) { + throw new DocumentSourceError("versionTooOld", response.status); + } + } + + async function tagMap(): Promise> { + const tags = await listTags(); + return new Map(tags.map((tag) => [tag.id, tag.name])); + } + + async function listTags(): Promise { + const query = new URLSearchParams({ + page_size: String(DOCUMENT_PICKER_MAX_TAGS), + ordering: "name", + }); + const { body, response } = await sourceJson(target, `/api/tags/?${query}`); + checkVersion(response); + const page = asRecord(body); + if (!page) throw shapeError(); + const tags: DocumentSourceTagDto[] = []; + for (const raw of asArray(page.results)) { + const tag = asRecord(raw); + const id = asText(tag?.id); + const name = asText(tag?.name); + if (id && name) tags.push({ id, name }); + } + return tags; + } + + return { + system: "PAPERLESS", + + async test() { + const { body, response } = await sourceJson( + target, + "/api/documents/?page_size=1&fields=id", + ); + checkVersion(response); + if (!asRecord(body)) throw shapeError(); + }, + + tags: listTags, + + async search(params) { + const query = new URLSearchParams({ + page: String(params.page), + page_size: String(DOCUMENT_PICKER_PAGE_SIZE), + ordering: "-created", + truncate_content: "true", + fields: "id,title,created,added,tags,original_file_name", + }); + if (params.q) query.set("title__icontains", params.q); + if (params.tagId) query.set("tags__id__all", params.tagId); + if (params.from) query.set("created__date__gte", params.from); + if (params.to) query.set("created__date__lte", params.to); + + let page: Json | null; + let names: Map; + try { + const [list, tags] = await Promise.all([ + sourceJson(target, `/api/documents/?${query}`), + tagMap(), + ]); + checkVersion(list.response); + page = asRecord(list.body); + names = tags; + } catch (err) { + // Paperless answers a page past the end with 404 ("Invalid page"): + // there is simply nothing more. + if (err instanceof DocumentSourceError && err.code === "notFound") { + return { items: [], hasMore: false }; + } + throw err; + } + if (!page) throw shapeError(); + + const items: SourceListItem[] = []; + for (const raw of asArray(page.results)) { + const doc = asRecord(raw); + const id = asText(doc?.id); + if (!doc || !id) continue; + const tagIds = asArray(doc.tags) + .map(asText) + .filter((t): t is string => t !== null); + if (params.tagId && !tagIds.includes(params.tagId)) continue; + const date = dayOf(doc.created) ?? dayOf(doc.added); + if (!withinRange(date, params.from, params.to)) continue; + items.push({ + sourceId: id, + title: titleOf( + asText(doc.title), + asText(doc.original_file_name), + `Paperless ${id}`, + ), + date, + tags: tagIds + .map((tagId) => names.get(tagId)) + .filter((n): n is string => typeof n === "string"), + sizeBytes: null, + }); + } + return { items, hasMore: typeof page.next === "string" && !!page.next }; + }, + + async document(sourceId) { + const { body, response } = await sourceJson( + target, + `/api/documents/${encodeURIComponent(sourceId)}/?fields=id,title,created,added,original_file_name`, + ); + checkVersion(response); + const doc = asRecord(body); + if (!doc) throw shapeError(); + const filename = asText(doc.original_file_name); + return { + title: titleOf(asText(doc.title), filename, `Paperless ${sourceId}`), + date: dayOf(doc.created) ?? dayOf(doc.added), + filename, + }; + }, + + download(sourceId, maxFileBytes) { + // The original, not the archived PDF/A: the same bytes a Paperless + // workflow or the import script sends, so they dedup against each other. + return sourceDownload( + download, + `/api/documents/${encodeURIComponent(sourceId)}/download/?original=true`, + maxFileBytes, + ); + }, + }; +} + +// ── Papra ────────────────────────────────────────────────────────────────── + +/** Papra's search syntax quotes a tag name that holds a space. */ +function papraTagTerm(name: string): string { + return /[\s"]/.test(name) ? `tag:${JSON.stringify(name)}` : `tag:${name}`; +} + +function papra(connection: SourceConnection): SourceClient { + if (!connection.organizationId) { + throw new DocumentSourceError("notFound"); + } + const org = `/api/organizations/${encodeURIComponent(connection.organizationId)}`; + const target: SourceTarget = { + origin: connection.origin, + baseUrl: connection.baseUrl, + headers: { + Authorization: `Bearer ${connection.token}`, + Accept: "application/json", + }, + }; + const download: SourceTarget = { + ...target, + headers: { Authorization: `Bearer ${connection.token}` }, + }; + + async function listTags(): Promise { + const { body } = await sourceJson(target, `${org}/tags`); + const page = asRecord(body); + if (!page) throw shapeError(); + const tags: DocumentSourceTagDto[] = []; + for (const raw of asArray(page.tags)) { + const tag = asRecord(raw); + const id = asText(tag?.id); + const name = asText(tag?.name); + if (id && name) tags.push({ id, name }); + } + return tags + .sort((a, b) => a.name.localeCompare(b.name)) + .slice(0, DOCUMENT_PICKER_MAX_TAGS); + } + + return { + system: "PAPRA", + + async test() { + const { body } = await sourceJson( + target, + `${org}/documents?pageIndex=0&pageSize=1`, + ); + if (!asRecord(body)) throw shapeError(); + }, + + tags: listTags, + + async search(params) { + const terms: string[] = []; + if (params.q) terms.push(params.q); + if (params.tagId) { + const tag = (await listTags()).find((t) => t.id === params.tagId); + // A tag that no longer exists matches nothing, rather than silently + // dropping the filter and listing everything. + if (!tag) return { items: [], hasMore: false }; + terms.push(papraTagTerm(tag.name)); + } + const pageIndex = params.page - 1; + const query = new URLSearchParams({ + pageIndex: String(pageIndex), + pageSize: String(DOCUMENT_PICKER_PAGE_SIZE), + }); + if (terms.length > 0) query.set("searchQuery", terms.join(" ")); + const { body } = await sourceJson(target, `${org}/documents?${query}`); + const page = asRecord(body); + if (!page) throw shapeError(); + + const documents = asArray(page.documents); + const items: SourceListItem[] = []; + for (const raw of documents) { + const doc = asRecord(raw); + const id = asText(doc?.id); + if (!doc || !id) continue; + const tags = asArray(doc.tags) + .map(asRecord) + .filter((t): t is Json => t !== null); + if (params.tagId && !tags.some((t) => asText(t.id) === params.tagId)) { + continue; + } + // Papra's search date syntax differs between versions; the range is + // applied here instead of trusting a query syntax. + const date = dayOf(doc.documentDate) ?? dayOf(doc.createdAt); + if (!withinRange(date, params.from, params.to)) continue; + const size = doc.originalSize; + items.push({ + sourceId: id, + title: titleOf( + stripExtension(asText(doc.name)), + asText(doc.originalName), + `Papra ${id}`, + ), + date, + tags: tags + .map((t) => asText(t.name)) + .filter((n): n is string => n !== null), + sizeBytes: typeof size === "number" && size >= 0 ? size : null, + }); + } + const total = + typeof page.documentsCount === "number" ? page.documentsCount : null; + const hasMore = + total !== null + ? (pageIndex + 1) * DOCUMENT_PICKER_PAGE_SIZE < total + : documents.length >= DOCUMENT_PICKER_PAGE_SIZE; + return { items, hasMore }; + }, + + async document(sourceId) { + const { body } = await sourceJson( + target, + `${org}/documents/${encodeURIComponent(sourceId)}`, + ); + const doc = asRecord(asRecord(body)?.document); + if (!doc) throw shapeError(); + const filename = asText(doc.originalName) ?? asText(doc.name); + return { + title: titleOf( + stripExtension(asText(doc.name)), + filename, + `Papra ${sourceId}`, + ), + date: dayOf(doc.documentDate) ?? dayOf(doc.createdAt), + filename, + }; + }, + + download(sourceId, maxFileBytes) { + return sourceDownload( + download, + `${org}/documents/${encodeURIComponent(sourceId)}/file`, + maxFileBytes, + ); + }, + }; +} + +export function sourceClient(connection: SourceConnection): SourceClient { + return connection.system === "PAPERLESS" + ? paperless(connection) + : papra(connection); +} diff --git a/src/lib/documents/sources/connections.ts b/src/lib/documents/sources/connections.ts new file mode 100644 index 000000000..c8f0dad2c --- /dev/null +++ b/src/lib/documents/sources/connections.ts @@ -0,0 +1,117 @@ +/** + * Reading a person's document-source connection, and the one place its token + * is decrypted. + * + * {@link toConnectionDto} is the only shape a connection leaves the server in, + * and it has no field that could carry the token: `hasToken` is the literal + * `true`. {@link openConnection} decrypts the token for exactly one request to + * the source and re-checks the stored origin against the operator's list on + * the way, so a connection whose origin was removed from + * `DOCUMENT_SOURCE_ORIGINS` answers `originNotAllowed` instead of dialling. + */ +import { decrypt } from "@/lib/crypto"; +import { prisma } from "@/lib/db"; + +import { sourceClient, type SourceClient } from "./clients"; +import { DocumentSourceError } from "./http"; +import { evaluateSourceBaseUrl } from "./origins"; +import type { + DocumentPickerSystem, + DocumentSourceConnectionDto, +} from "./types"; + +/** The columns a DTO or a request needs; never the whole row by accident. */ +const CONNECTION_SELECT = { + system: true, + baseUrl: true, + organizationId: true, + tokenEncrypted: true, + lastVerifiedAt: true, +} as const; + +export interface StoredConnection { + system: string; + baseUrl: string; + organizationId: string | null; + tokenEncrypted: string; + lastVerifiedAt: Date | null; +} + +export function toConnectionDto( + row: Omit, +): DocumentSourceConnectionDto { + return { + system: row.system as DocumentPickerSystem, + baseUrl: row.baseUrl, + organizationId: row.organizationId, + hasToken: true, + lastVerifiedAt: row.lastVerifiedAt?.toISOString() ?? null, + originAllowed: evaluateSourceBaseUrl(row.baseUrl).ok, + }; +} + +export function listConnections( + userId: string, +): Promise[]> { + return prisma.documentSourceConnection.findMany({ + where: { userId }, + select: { + system: true, + baseUrl: true, + organizationId: true, + lastVerifiedAt: true, + }, + orderBy: { system: "asc" }, + }); +} + +export function loadConnection( + userId: string, + system: DocumentPickerSystem, +): Promise { + return prisma.documentSourceConnection.findUnique({ + where: { userId_system: { userId, system } }, + select: CONNECTION_SELECT, + }); +} + +/** + * A client for one of the person's stored connections, or the reason there is + * none. Throws {@link DocumentSourceError}: `notConnected` when nothing is + * stored for the system, `originNotAllowed` when the operator no longer lists + * the stored origin. + */ +export async function openConnection( + userId: string, + system: DocumentPickerSystem, +): Promise { + const row = await loadConnection(userId, system); + if (!row) throw new DocumentSourceError("notConnected"); + return clientFor(row); +} + +/** A client for a row already in hand, with the origin re-checked. */ +export function clientFor(row: StoredConnection): SourceClient { + const verdict = evaluateSourceBaseUrl(row.baseUrl); + if (!verdict.ok) throw new DocumentSourceError("originNotAllowed"); + return sourceClient({ + system: row.system as DocumentPickerSystem, + origin: verdict.origin, + baseUrl: verdict.baseUrl, + organizationId: row.organizationId, + token: decrypt(row.tokenEncrypted), + }); +} + +/** Stamp a successful test. */ +export async function markVerified( + userId: string, + system: DocumentPickerSystem, +): Promise { + const at = new Date(); + await prisma.documentSourceConnection.updateMany({ + where: { userId, system }, + data: { lastVerifiedAt: at }, + }); + return at; +} diff --git a/src/lib/documents/sources/http.ts b/src/lib/documents/sources/http.ts new file mode 100644 index 000000000..473d442ea --- /dev/null +++ b/src/lib/documents/sources/http.ts @@ -0,0 +1,198 @@ +/** + * The document picker's one way to the network. + * + * Every request to a Paperless-ngx or Papra instance goes through + * {@link sourceRequest}, which goes through `safeFetch` with the connection's + * origin as `operatorApprovedPrivateOrigin`: + * + * - the target must have exactly that origin (a path, query or `next` link + * that names another host is refused before a socket opens); + * - redirects are never followed; a 3xx is reported as `redirected`, so a + * source cannot bounce the request and its `Authorization` header to + * another host; + * - the connect goes through the pinned operator-approved dispatcher, which + * refuses link-local, metadata and unspecified addresses at dial time even + * for a listed name; + * - every call has a timeout, and every body is read with a byte cap. + * + * The origin handed in is the one the caller just re-checked against + * `DOCUMENT_SOURCE_ORIGINS`. Nothing here decides trust; it only refuses to + * go anywhere else. + */ +import { safeFetch, SafeFetchError } from "@/lib/safe-fetch"; +import { BodyTooLargeError, readBoundedBody } from "@/lib/labs/ocr-upload"; +import { annotate } from "@/lib/logging/context"; + +import type { DocumentSourceErrorCode } from "./types"; + +/** JSON bodies larger than this are not a search page; refuse them. */ +export const SOURCE_JSON_MAX_BYTES = 1024 * 1024; +export const SOURCE_JSON_TIMEOUT_MS = 10_000; +export const SOURCE_DOWNLOAD_TIMEOUT_MS = 60_000; + +/** + * A request to a source ended somewhere other than a usable answer. The code + * is what the client translates; `upstreamStatus` is the only thing about the + * source's answer that is passed on. Its body never is: an error page can + * echo the request's credentials or internal paths. + */ +export class DocumentSourceError extends Error { + readonly code: DocumentSourceErrorCode; + readonly upstreamStatus?: number; + + constructor(code: DocumentSourceErrorCode, upstreamStatus?: number) { + super(`document source: ${code}`); + this.name = "DocumentSourceError"; + this.code = code; + if (upstreamStatus !== undefined) this.upstreamStatus = upstreamStatus; + } +} + +/** The document is larger than the vault accepts; carries the cap. */ +export class DocumentSourceTooLargeError extends Error { + constructor(readonly maxFileBytes: number) { + super("document source: file larger than the vault accepts"); + this.name = "DocumentSourceTooLargeError"; + } +} + +export interface SourceTarget { + /** The listed origin, exactly as `DOCUMENT_SOURCE_ORIGINS` holds it. */ + origin: string; + /** Origin plus the optional base path, no trailing slash. */ + baseUrl: string; + headers: Record; +} + +async function sourceRequest( + target: SourceTarget, + path: string, + timeoutMs: number, +): Promise { + const url = `${target.baseUrl}${path}`; + let response: Response; + try { + response = await safeFetch( + url, + { method: "GET", headers: target.headers, redirect: "manual" }, + { operatorApprovedPrivateOrigin: target.origin, timeoutMs }, + ); + } catch (err) { + // A refused destination, a timeout and a dead host all read the same to + // the person: the source could not be reached from here. The operator's + // wide event keeps the kind. + annotate({ + meta: { + sourceFailure: err instanceof SafeFetchError ? err.kind : "network", + }, + }); + throw new DocumentSourceError("unreachable"); + } + if (response.status >= 300 && response.status < 400) { + await discard(response); + throw new DocumentSourceError("redirected", response.status); + } + return response; +} + +async function discard(response: Response): Promise { + try { + await response.body?.cancel(); + } catch { + // nothing to release + } +} + +/** Map a non-2xx answer to the error the person sees. */ +export async function failFor( + response: Response, + notFound: DocumentSourceErrorCode = "notFound", +): Promise { + await discard(response); + const status = response.status; + if (status === 401 || status === 403) { + throw new DocumentSourceError("authRefused", status); + } + if (status === 404) throw new DocumentSourceError(notFound, status); + // Paperless-ngx answers 406 to an API version it no longer (or not yet) + // speaks. + if (status === 406) throw new DocumentSourceError("versionTooOld", status); + throw new DocumentSourceError("badResponse", status); +} + +/** GET a JSON document, bounded in time and size. */ +export async function sourceJson( + target: SourceTarget, + path: string, + options: { notFound?: DocumentSourceErrorCode } = {}, +): Promise<{ body: unknown; response: Response }> { + const response = await sourceRequest(target, path, SOURCE_JSON_TIMEOUT_MS); + if (!response.ok) return failFor(response, options.notFound); + const declared = Number(response.headers.get("content-length") ?? 0); + if (declared > SOURCE_JSON_MAX_BYTES) { + await discard(response); + throw new DocumentSourceError("badResponse", response.status); + } + let bytes: Uint8Array; + try { + bytes = await readBoundedBody(response.body, SOURCE_JSON_MAX_BYTES); + } catch { + throw new DocumentSourceError("badResponse", response.status); + } + try { + return { + body: JSON.parse(new TextDecoder().decode(bytes)) as unknown, + response, + }; + } catch { + // A login page or a proxy's HTML error instead of the API: most often a + // base address that points at the web app rather than its API root. + throw new DocumentSourceError("badResponse", response.status); + } +} + +/** GET a file, bounded by the vault's own per-file cap. */ +export async function sourceDownload( + target: SourceTarget, + path: string, + maxFileBytes: number, +): Promise<{ bytes: Uint8Array; filename: string | null }> { + const response = await sourceRequest( + target, + path, + SOURCE_DOWNLOAD_TIMEOUT_MS, + ); + if (!response.ok) return failFor(response); + const declared = Number(response.headers.get("content-length") ?? 0); + if (declared > maxFileBytes) { + await discard(response); + throw new DocumentSourceTooLargeError(maxFileBytes); + } + try { + const bytes = await readBoundedBody(response.body, maxFileBytes); + return { + bytes, + filename: filenameFrom(response.headers.get("content-disposition")), + }; + } catch (err) { + if (err instanceof BodyTooLargeError) { + throw new DocumentSourceTooLargeError(maxFileBytes); + } + throw new DocumentSourceError("unreachable"); + } +} + +/** The file name a download names, if it names one. */ +export function filenameFrom(header: string | null): string | null { + if (!header) return null; + const star = /filename\*=(?:UTF-8'')?([^;]+)/i.exec(header); + if (star) { + try { + return decodeURIComponent(star[1].trim().replace(/^"|"$/g, "")); + } catch { + // fall through to the plain form + } + } + const plain = /filename="?([^";]+)"?/i.exec(header); + return plain ? plain[1] : null; +} diff --git a/src/lib/documents/sources/origins.ts b/src/lib/documents/sources/origins.ts new file mode 100644 index 000000000..6d7bb16a2 --- /dev/null +++ b/src/lib/documents/sources/origins.ts @@ -0,0 +1,95 @@ +/** + * `DOCUMENT_SOURCE_ORIGINS` (#1038) — the operator grant that lets the + * document picker reach a Paperless-ngx or Papra instance. + * + * DOCUMENT_SOURCE_ORIGINS="https://paperless.example.com,http://papra.lan:1221" + * + * The grammar is the one every private-origin list shares + * (`src/lib/private-origin-policy.ts`): exact `scheme://host[:port]` origins, + * no path, query, credentials, wildcard, CIDR or suffix. A malformed entry is + * logged once, reduced to scheme and host, and grants nothing; the valid + * entries beside it keep working. + * + * Unlike the notification list, this one is the WHOLE trust decision, not an + * exception on top of a public-host default. An origin that is not listed is + * refused even when it is a public address, and with the variable unset the + * picker does not exist: the routes answer 404 and the client renders no + * button. A person can only ever point their connection at a system the + * operator named. Every request to a listed origin still goes through + * `safeFetch` with the origin pinned, redirects forbidden, and the dial-time + * floor that refuses link-local, metadata and unspecified addresses even for + * a listed name. + * + * Read directly from `process.env` so the compose-whitelist guard sees it, + * parsed once per distinct value, and re-read on every use: an operator who + * removes an origin cuts off the stored connections that name it on their + * next request. + */ +import { + parsePrivateOrigins, + redactGrantEntry, +} from "@/lib/private-origin-policy"; + +const ENV_NAME = "DOCUMENT_SOURCE_ORIGINS"; + +let cachedRaw: string | undefined; +let cachedOrigins: ReadonlySet = new Set(); + +export function configuredDocumentSourceOrigins(): ReadonlySet { + const raw = process.env.DOCUMENT_SOURCE_ORIGINS; + if (raw === cachedRaw) return cachedOrigins; + cachedOrigins = parsePrivateOrigins(raw, (redactedEntry, reason) => { + console.warn(`${ENV_NAME}: ignoring entry "${redactedEntry}": ${reason}`); + }); + cachedRaw = raw; + return cachedOrigins; +} + +/** Is the picker switched on for this instance at all? */ +export function documentSourcesAvailable(): boolean { + return configuredDocumentSourceOrigins().size > 0; +} + +/** Longest base URL a connection may store. */ +export const DOCUMENT_SOURCE_BASE_URL_MAX = 2048; + +export type SourceBaseVerdict = + | { ok: true; origin: string; baseUrl: string } + | { ok: false; reason: "invalid" | "notAllowed" }; + +/** + * Validate a base URL a person typed against the operator's list. + * + * The base may carry a path (a Paperless served under `/paperless`); only its + * origin is compared, and it must be listed exactly. Userinfo, a query and a + * fragment are refused: none of them belongs in a base address, and each is + * where a pasted credential would sit. Trailing slashes are trimmed so the + * client can append `/api/...` without doubling one. + */ +export function evaluateSourceBaseUrl( + value: string, + origins: ReadonlySet = configuredDocumentSourceOrigins(), +): SourceBaseVerdict { + let url: URL; + try { + url = new URL(value.trim()); + } catch { + return { ok: false, reason: "invalid" }; + } + if (url.protocol !== "http:" && url.protocol !== "https:") { + return { ok: false, reason: "invalid" }; + } + if (url.username || url.password || url.search || url.hash) { + return { ok: false, reason: "invalid" }; + } + if (!origins.has(url.origin)) return { ok: false, reason: "notAllowed" }; + const path = url.pathname.replace(/\/+$/, ""); + return { ok: true, origin: url.origin, baseUrl: `${url.origin}${path}` }; +} + +/** The listed origins as the settings card may show them. */ +export function listedDocumentSourceOrigins(): string[] { + return [...configuredDocumentSourceOrigins()] + .map((origin) => redactGrantEntry(origin)) + .sort(); +} diff --git a/src/lib/documents/sources/route-support.ts b/src/lib/documents/sources/route-support.ts new file mode 100644 index 000000000..295944b8c --- /dev/null +++ b/src/lib/documents/sources/route-support.ts @@ -0,0 +1,156 @@ +/** + * What every document-source route does before its own work, and how a source + * failure becomes a response. + * + * The admission rule, with {@link admitDocumentSourceCaller} as its second + * half: + * + * - the route's own `requireAuth()` with no scope: a session, or a `["*"]` token; a narrow + * token is refused, and so is any request acting on somebody else's record + * (403 `sharing.not_permitted`). A delegate, whatever their grant, and a + * guardian acting for a managed record never reach a connection: it is the + * owner's credential to a system outside HealthLog, and driving it would + * read the owner's whole archive there, which no grant covers; + * - then the transport must be the cookie. A `["*"]` Bearer (the native + * client, a script holding a login token) is refused: the picker is a + * browser surface, and a credential for another system is managed only + * from one; + * - the documents module must be on, and the operator must have listed at + * least one origin in `DOCUMENT_SOURCE_ORIGINS`. Without the list the + * routes answer 404 before reading a row. + */ +import type { AuthContext } from "@/lib/api-handler"; +import { apiError } from "@/lib/api-response"; +import { requireModuleEnabled } from "@/lib/modules/gate"; +import { checkRateLimit, type RateLimitResult } from "@/lib/rate-limit"; + +import { DocumentSourceError, DocumentSourceTooLargeError } from "./http"; +import { documentSourcesAvailable } from "./origins"; +import { + documentSourceErrorCode, + systemFromSlug, + type DocumentPickerSystem, + type DocumentSourceErrorCode, +} from "./types"; + +/** + * Admit a caller the route has already resolved with a bare `requireAuth()`. + * + * Each route calls `requireAuth()` itself, in its own module, so the sharing + * guard's scan sees every one of them as a bare, owner-only route rather than + * a helper it cannot follow. This adds the rest: a cookie transport, the + * documents module, and (unless `needsList` is false, which only the status + * read passes) the operator's list. + */ +export async function admitDocumentSourceCaller( + auth: AuthContext, + options: { needsList?: boolean } = {}, +): Promise { + if (auth.authMethod !== "cookie") { + return apiError( + "Document archives are managed from a browser session.", + 403, + { errorCode: "documents.sources.browserOnly" }, + ); + } + const gate = await requireModuleEnabled(auth.user.id, "inboundDocuments"); + if (!gate.enabled) return gate.response; + if (options.needsList !== false && !documentSourcesAvailable()) { + return sourceErrorResponse("unavailable"); + } + return null; +} + +/** The `{system}` path segment, or a 404 for anything else. */ +export async function systemParam( + params: Promise<{ system: string }>, +): Promise { + const { system } = await params; + const resolved = systemFromSlug(system); + return resolved ?? apiError("Unknown document system.", 404); +} + +const STATUS: Record = { + unavailable: 404, + notConnected: 404, + originNotAllowed: 422, + unreachable: 502, + redirected: 502, + authRefused: 502, + versionTooOld: 502, + badResponse: 502, + notFound: 404, + linkTargetNotFound: 404, + rateLimited: 429, +}; + +const MESSAGE: Record = { + unavailable: "Document archives are not enabled on this server.", + notConnected: "This document archive is not connected.", + originNotAllowed: + "This address is not on the server's list of allowed document archives.", + unreachable: "The document archive could not be reached.", + redirected: + "The document archive redirected the request. Use the address it redirects to.", + authRefused: "The document archive refused the API token.", + versionTooOld: "The document archive's version is too old.", + badResponse: "The document archive gave an answer HealthLog cannot read.", + notFound: "The document archive has no such document or organization.", + linkTargetNotFound: "The record to link to was not found.", + rateLimited: "Too many requests. Try again later.", +}; + +export function sourceErrorResponse( + code: DocumentSourceErrorCode, + upstreamStatus?: number, + headers?: Record, +): Response { + return apiError(MESSAGE[code], STATUS[code], { + errorCode: documentSourceErrorCode(code), + ...(upstreamStatus !== undefined ? { upstreamStatus } : {}), + ...(headers ? { headers } : {}), + }); +} + +/** Map a thrown source failure to its response; rethrow anything else. */ +export function responseForSourceFailure(err: unknown): Response { + if (err instanceof DocumentSourceError) { + return sourceErrorResponse(err.code, err.upstreamStatus); + } + if (err instanceof DocumentSourceTooLargeError) { + return apiError("File is too large.", 413, { + errorCode: "documents.inbound.fileTooLarge", + reason: "fileTooLarge", + maxFileBytes: err.maxFileBytes, + }); + } + throw err; +} + +/** Saves and tests share one allowance: each one dials the source. */ +const CONNECT_LIMIT_PER_MINUTE = 10; + +export function checkConnectRateLimit( + userId: string, +): Promise { + return checkRateLimit( + `documents-source-connect:${userId}`, + CONNECT_LIMIT_PER_MINUTE, + 60_000, + ); +} + +/** + * Searches and tag reads per minute. Each one is a request to the source; a + * debounced search field sends one per pause in typing, which this leaves + * ample room for. + */ +const SEARCH_LIMIT_PER_MINUTE = 60; + +export function checkSearchRateLimit(userId: string): Promise { + return checkRateLimit( + `documents-source-search:${userId}`, + SEARCH_LIMIT_PER_MINUTE, + 60_000, + ); +} diff --git a/src/lib/documents/sources/types.ts b/src/lib/documents/sources/types.ts new file mode 100644 index 000000000..187f670b3 --- /dev/null +++ b/src/lib/documents/sources/types.ts @@ -0,0 +1,127 @@ +/** + * Shared vocabulary of the document picker (#1038): which systems it speaks, + * the minimal shapes it hands to the client, and the errors a source can end + * in. A zero-dependency leaf so the client components, the OpenAPI registry + * and the routes import one list. + */ + +/** The systems the picker can search and import from. */ +export const DOCUMENT_PICKER_SYSTEMS = ["PAPERLESS", "PAPRA"] as const; +export type DocumentPickerSystem = (typeof DOCUMENT_PICKER_SYSTEMS)[number]; + +/** The lowercase path segment for each system (`/api/documents/sources/{slug}`). */ +export const DOCUMENT_PICKER_SLUGS = { + paperless: "PAPERLESS", + papra: "PAPRA", +} as const satisfies Record; +export type DocumentPickerSlug = keyof typeof DOCUMENT_PICKER_SLUGS; + +export function systemFromSlug( + slug: string | undefined, +): DocumentPickerSystem | null { + return slug && Object.hasOwn(DOCUMENT_PICKER_SLUGS, slug) + ? DOCUMENT_PICKER_SLUGS[slug as DocumentPickerSlug] + : null; +} + +export function slugForSystem( + system: DocumentPickerSystem, +): DocumentPickerSlug { + return system === "PAPERLESS" ? "paperless" : "papra"; +} + +/** Results per search page. */ +export const DOCUMENT_PICKER_PAGE_SIZE = 25; +/** Highest page a search may ask for. */ +export const DOCUMENT_PICKER_MAX_PAGE = 500; +/** Longest name query. */ +export const DOCUMENT_PICKER_QUERY_MAX = 200; +/** Documents the sheet imports in one run. */ +export const DOCUMENT_PICKER_MAX_SELECTION = 25; +/** Tags offered in the filter. */ +export const DOCUMENT_PICKER_MAX_TAGS = 250; + +/** A connection as every route may show it: never the token. */ +export interface DocumentSourceConnectionDto { + system: DocumentPickerSystem; + baseUrl: string; + organizationId: string | null; + hasToken: true; + lastVerifiedAt: string | null; + /** False once the operator removed the origin from the list. */ + originAllowed: boolean; +} + +export interface DocumentSourcesStatusDto { + /** The operator listed at least one origin; otherwise the picker is off. */ + available: boolean; + /** The listed origins, so the settings card can say what is allowed. */ + allowedOrigins: string[]; + connections: DocumentSourceConnectionDto[]; +} + +export interface DocumentSourceTagDto { + id: string; + name: string; +} + +/** Whether HealthLog already holds a result, by its source key. */ +export type DocumentPickerResultState = "new" | "imported" | "deleted"; + +export interface DocumentSourceResultDto { + sourceId: string; + title: string; + /** YYYY-MM-DD, the document's own date in the source, or null. */ + date: string | null; + tags: string[]; + sizeBytes: number | null; + state: DocumentPickerResultState; + /** The vault document when `state` is `imported`. */ + documentId: string | null; +} + +export interface DocumentSourceSearchDto { + results: DocumentSourceResultDto[]; + page: number; + hasMore: boolean; +} + +export type DocumentImportOutcome = "imported" | "duplicate" | "deleted"; + +export interface DocumentSourceImportDto { + outcome: DocumentImportOutcome; + /** The vault document; null only when a deleted one has been purged. */ + documentId: string | null; + /** The document was linked to the record the picker was opened from. */ + linked: boolean; +} + +/** Record kinds a picked document can be linked to on import. */ +export const DOCUMENT_PICKER_LINK_KINDS = [ + "conditionEpisode", + "encounter", + "vaccination", +] as const; +export type DocumentPickerLinkKind = + (typeof DOCUMENT_PICKER_LINK_KINDS)[number]; + +/** Why a request to a source failed, as `meta.errorCode` suffixes. */ +export const DOCUMENT_SOURCE_ERROR_CODES = [ + "unavailable", + "notConnected", + "originNotAllowed", + "unreachable", + "redirected", + "authRefused", + "versionTooOld", + "badResponse", + "notFound", + "linkTargetNotFound", + "rateLimited", +] as const; +export type DocumentSourceErrorCode = + (typeof DOCUMENT_SOURCE_ERROR_CODES)[number]; + +export function documentSourceErrorCode(code: DocumentSourceErrorCode): string { + return `documents.sources.${code}`; +} diff --git a/src/lib/export/backup-plan.ts b/src/lib/export/backup-plan.ts index c9fa7b80a..2d689337a 100644 --- a/src/lib/export/backup-plan.ts +++ b/src/lib/export/backup-plan.ts @@ -620,6 +620,8 @@ export const NOT_IN_BACKUP_MODELS: Readonly> = { "A CSRF nonce for a handshake that is valid for minutes and is already over by the time any backup is read.", McpOAuthConnection: "A grant issued to a client that was registered against this instance.", + DocumentSourceConnection: + "A person's API token for their Paperless-ngx or Papra instance. A backup file is not a credential store, same reasoning as ApiToken; and restored on another host it would point at an address that host's operator never allowed in DOCUMENT_SOURCE_ORIGINS. The documents imported through it are carried with the vault; reconnecting takes one form.", IntegrationStatus: "A ledger of what this deployment's syncs did. Restoring it onto another host would assert a sync history that host never had.", diff --git a/src/lib/validations/document-sources.ts b/src/lib/validations/document-sources.ts new file mode 100644 index 000000000..6997cb97d --- /dev/null +++ b/src/lib/validations/document-sources.ts @@ -0,0 +1,91 @@ +/** + * Request schemas of the document picker (#1038): saving a connection, + * searching a source, importing one picked document. No `userId` field in any + * of them; the owner is always the authenticated session. + */ +import { z } from "zod/v4"; + +import { DOCUMENT_SOURCE_BASE_URL_MAX } from "@/lib/documents/sources/origins"; +import { + DOCUMENT_PICKER_LINK_KINDS, + DOCUMENT_PICKER_MAX_PAGE, + DOCUMENT_PICKER_QUERY_MAX, +} from "@/lib/documents/sources/types"; +import { + documentSourceIdSchema, + INBOUND_DOCUMENT_KINDS, +} from "@/lib/validations/inbound-documents"; + +/** Printable ASCII without spaces: an id or a key, never prose. */ +const printable = /^[\x21-\x7e]+$/u; + +const isoDate = z + .string() + .regex(/^\d{4}-\d{2}-\d{2}$/u, "Expected a YYYY-MM-DD date"); + +/** Longest API token or key a connection stores. */ +export const DOCUMENT_SOURCE_TOKEN_MAX = 512; + +/** + * Save a connection. `token` may be left out when a connection for the system + * already exists: the stored one is kept, so the address or the organization + * can change without typing the key again. The first save must carry it (the + * route says so). `organizationId` is Papra's and required there. + */ +export const documentSourceSaveSchema = z.object({ + baseUrl: z.string().trim().min(1).max(DOCUMENT_SOURCE_BASE_URL_MAX), + organizationId: z + .string() + .trim() + .min(1) + .max(128) + .regex(printable, "Expected printable characters without spaces") + .optional(), + token: z + .string() + .trim() + .min(1) + .max(DOCUMENT_SOURCE_TOKEN_MAX) + .regex(printable, "Expected printable characters without spaces") + .optional(), +}); +export type DocumentSourceSaveInput = z.infer; + +export const documentSourceSearchSchema = z + .object({ + q: z.string().trim().max(DOCUMENT_PICKER_QUERY_MAX).default(""), + tag: z + .string() + .trim() + .min(1) + .max(64) + .regex(printable, "Expected printable characters without spaces") + .optional(), + from: isoDate.optional(), + to: isoDate.optional(), + page: z.coerce + .number() + .int() + .min(1) + .max(DOCUMENT_PICKER_MAX_PAGE) + .default(1), + }) + .refine((v) => !v.from || !v.to || v.from <= v.to, { + path: ["to"], + message: "The end date is before the start date", + }); + +export const documentSourceImportSchema = z.object({ + sourceId: documentSourceIdSchema, + kind: z.enum(INBOUND_DOCUMENT_KINDS).optional(), + /** The record the picker was opened from, to link the document to. */ + link: z + .object({ + kind: z.enum(DOCUMENT_PICKER_LINK_KINDS), + id: z.string().trim().min(1).max(64), + }) + .optional(), +}); +export type DocumentSourceImportInput = z.infer< + typeof documentSourceImportSchema +>; diff --git a/src/lib/validations/inbound-documents.ts b/src/lib/validations/inbound-documents.ts index cd78d826a..ebc8aa030 100644 --- a/src/lib/validations/inbound-documents.ts +++ b/src/lib/validations/inbound-documents.ts @@ -701,7 +701,7 @@ export const DOCUMENT_SOURCE_ID_MAX = 128; * The document's id in its source system. Printable ASCII only: it is a key, * not prose, and it is echoed on the detail sheet. */ -const documentSourceIdSchema = z +export const documentSourceIdSchema = z .string() .trim() .min(1) From cab67b4c066ecf7cc440f4ad5b211805f99d95c6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Sat, 26 Sep 2026 13:03:45 +0200 Subject: [PATCH 03/18] test(documents): pin the picker's origin allowlist and both source adapters --- .../sources/__tests__/clients.test.ts | 435 ++++++++++++++++++ .../sources/__tests__/origins.test.ts | 110 +++++ 2 files changed, 545 insertions(+) create mode 100644 src/lib/documents/sources/__tests__/clients.test.ts create mode 100644 src/lib/documents/sources/__tests__/origins.test.ts diff --git a/src/lib/documents/sources/__tests__/clients.test.ts b/src/lib/documents/sources/__tests__/clients.test.ts new file mode 100644 index 000000000..5bc780029 --- /dev/null +++ b/src/lib/documents/sources/__tests__/clients.test.ts @@ -0,0 +1,435 @@ +/** + * The Paperless-ngx and Papra adapters against recorded response shapes, with + * the network mocked at `safeFetch`. What these pin: + * + * - every request carries the listed origin as the operator-approved pin and + * forbids redirects, and a 3xx is reported, not followed; + * - the source's own query parameters (name, tag, date, page), and the local + * re-check that keeps a server ignoring a filter from widening a result; + * - the error mapping (401 → authRefused, 406 / an old X-Api-Version → + * versionTooOld, 404 → notFound, non-JSON → badResponse, a refused or dead + * host → unreachable); + * - the byte caps on JSON and on a download. + */ +import { beforeEach, describe, expect, it, vi } from "vitest"; + +const safeFetch = vi.hoisted(() => vi.fn()); + +vi.mock("@/lib/safe-fetch", async (importOriginal) => { + const actual = await importOriginal(); + return { ...actual, safeFetch }; +}); + +import { SafeFetchError } from "@/lib/safe-fetch"; + +import { sourceClient } from "../clients"; +import { + DocumentSourceError, + DocumentSourceTooLargeError, + SOURCE_JSON_MAX_BYTES, +} from "../http"; + +const PAPERLESS = { + system: "PAPERLESS" as const, + origin: "http://paperless.lan:8000", + baseUrl: "http://paperless.lan:8000/paperless", + organizationId: null, + token: "pl-secret-token", +}; + +const PAPRA = { + system: "PAPRA" as const, + origin: "https://papra.example.com", + baseUrl: "https://papra.example.com", + organizationId: "org_123", + token: "ppk_secret", +}; + +function json(body: unknown, init: ResponseInit = {}): Response { + return new Response(JSON.stringify(body), { + status: 200, + headers: { "content-type": "application/json" }, + ...init, + }); +} + +/** Route mocked requests by path; anything unmatched is a 404. */ +function serve(routes: Record Response>) { + safeFetch.mockImplementation(async (target: string) => { + const url = new URL(target); + for (const [prefix, handler] of Object.entries(routes)) { + if (url.pathname.startsWith(prefix)) return handler(url); + } + return new Response("not found", { status: 404 }); + }); +} + +function calledUrls(): URL[] { + return safeFetch.mock.calls.map((c) => new URL(String(c[0]))); +} + +async function failure(promise: Promise) { + try { + await promise; + } catch (err) { + return err; + } + throw new Error("expected a failure"); +} + +beforeEach(() => { + safeFetch.mockReset(); +}); + +describe("every request is pinned to the listed origin", () => { + it("passes the origin as the operator-approved pin and forbids redirects", async () => { + serve({ "/paperless/api/documents/": () => json({ results: [] }) }); + await sourceClient(PAPERLESS).test(); + expect(safeFetch).toHaveBeenCalledTimes(1); + const [target, init, opts] = safeFetch.mock.calls[0]; + expect(String(target)).toBe( + "http://paperless.lan:8000/paperless/api/documents/?page_size=1&fields=id", + ); + expect(init.redirect).toBe("manual"); + expect(init.headers.Authorization).toBe("Token pl-secret-token"); + expect(init.headers.Accept).toBe("application/json; version=9"); + expect(opts.operatorApprovedPrivateOrigin).toBe( + "http://paperless.lan:8000", + ); + expect(opts.timeoutMs).toBeGreaterThan(0); + expect(opts.requirePublicHost).toBeUndefined(); + }); + + it("reports a redirect instead of following it", async () => { + serve({ + "/paperless/api/documents/": () => + new Response(null, { + status: 302, + headers: { location: "http://169.254.169.254/latest/meta-data" }, + }), + }); + const err = await failure(sourceClient(PAPERLESS).test()); + expect(err).toBeInstanceOf(DocumentSourceError); + expect((err as DocumentSourceError).code).toBe("redirected"); + expect(safeFetch).toHaveBeenCalledTimes(1); + }); + + it("reports a refused or dead destination as unreachable", async () => { + safeFetch.mockRejectedValue(new SafeFetchError("refused", "private_host")); + const err = await failure(sourceClient(PAPRA).test()); + expect((err as DocumentSourceError).code).toBe("unreachable"); + }); +}); + +describe("Paperless-ngx", () => { + const tags = { + results: [ + { id: 7, name: "Health" }, + { id: 9, name: "Insurance" }, + ], + next: null, + }; + + it("searches by title with tag, date and page, and names the tags", async () => { + serve({ + "/paperless/api/tags/": () => json(tags), + "/paperless/api/documents/": () => + json({ + count: 30, + next: "http://paperless.lan:8000/paperless/api/documents/?page=3", + results: [ + { + id: 41, + title: "Blood test", + created: "2025-03-04", + tags: [7], + original_file_name: "labor.pdf", + }, + // A server that ignored the tag filter must not widen the result. + { id: 42, title: "Car", created: "2025-03-05", tags: [9] }, + // Nor one that ignored the date filter. + { id: 43, title: "Old", created: "2019-01-01", tags: [7] }, + // No title: the file name without its extension. + { + id: 44, + title: " ", + created: "2025-06-01T10:00:00+02:00", + tags: [7], + original_file_name: "discharge-letter.pdf", + }, + ], + }), + }); + const result = await sourceClient(PAPERLESS).search({ + q: "blood", + tagId: "7", + from: "2025-01-01", + to: "2025-12-31", + page: 2, + }); + + const list = calledUrls().find((u) => u.pathname.endsWith("/documents/")); + expect(list?.searchParams.get("title__icontains")).toBe("blood"); + expect(list?.searchParams.get("tags__id__all")).toBe("7"); + expect(list?.searchParams.get("created__date__gte")).toBe("2025-01-01"); + expect(list?.searchParams.get("created__date__lte")).toBe("2025-12-31"); + expect(list?.searchParams.get("page")).toBe("2"); + expect(list?.searchParams.get("page_size")).toBe("25"); + + expect(result.hasMore).toBe(true); + expect(result.items).toEqual([ + { + sourceId: "41", + title: "Blood test", + date: "2025-03-04", + tags: ["Health"], + sizeBytes: null, + }, + { + sourceId: "44", + title: "discharge-letter", + date: "2025-06-01", + tags: ["Health"], + sizeBytes: null, + }, + ]); + }); + + it("answers a page past the end with nothing", async () => { + serve({ + "/paperless/api/tags/": () => json(tags), + "/paperless/api/documents/": () => + json({ detail: "Invalid page." }, { status: 404 }), + }); + await expect( + sourceClient(PAPERLESS).search({ + q: "", + tagId: null, + from: null, + to: null, + page: 99, + }), + ).resolves.toEqual({ items: [], hasMore: false }); + }); + + it("maps a refused token, an unsupported version and an old server", async () => { + serve({ "/paperless/api/": () => json({}, { status: 401 }) }); + expect( + ((await failure(sourceClient(PAPERLESS).test())) as DocumentSourceError) + .code, + ).toBe("authRefused"); + + serve({ "/paperless/api/": () => json({}, { status: 406 }) }); + expect( + ((await failure(sourceClient(PAPERLESS).test())) as DocumentSourceError) + .code, + ).toBe("versionTooOld"); + + serve({ + "/paperless/api/": () => + json({ results: [] }, { headers: { "x-api-version": "5" } }), + }); + expect( + ((await failure(sourceClient(PAPERLESS).test())) as DocumentSourceError) + .code, + ).toBe("versionTooOld"); + }); + + it("reports a web page where the API should be as a bad response", async () => { + serve({ + "/paperless/api/": () => + new Response("login", { + headers: { "content-type": "text/html" }, + }), + }); + const err = await failure(sourceClient(PAPERLESS).test()); + expect((err as DocumentSourceError).code).toBe("badResponse"); + }); + + it("refuses a JSON answer larger than the cap", async () => { + serve({ + "/paperless/api/": () => + new Response("x".repeat(SOURCE_JSON_MAX_BYTES + 10), { status: 200 }), + }); + const err = await failure(sourceClient(PAPERLESS).test()); + expect((err as DocumentSourceError).code).toBe("badResponse"); + }); + + it("reads a document's details and downloads the original, within the cap", async () => { + serve({ + "/paperless/api/documents/41/download/": () => + new Response(new Uint8Array([37, 80, 68, 70]), { + headers: { "content-disposition": 'attachment; filename="x.pdf"' }, + }), + "/paperless/api/documents/41/": () => + json({ + id: 41, + title: "Blood test", + created: "2025-03-04", + original_file_name: "labor.pdf", + }), + }); + const client = sourceClient(PAPERLESS); + await expect(client.document("41")).resolves.toEqual({ + title: "Blood test", + date: "2025-03-04", + filename: "labor.pdf", + }); + const file = await client.download("41", 1024); + expect([...file.bytes]).toEqual([37, 80, 68, 70]); + expect(file.filename).toBe("x.pdf"); + const download = calledUrls().at(-1); + expect(download?.searchParams.get("original")).toBe("true"); + }); + + it("refuses a download larger than the vault accepts, declared or not", async () => { + serve({ + "/paperless/api/documents/41/download/": () => + new Response(new Uint8Array(2048), { + headers: { "content-length": "2048" }, + }), + }); + expect( + await failure(sourceClient(PAPERLESS).download("41", 1024)), + ).toBeInstanceOf(DocumentSourceTooLargeError); + + serve({ + "/paperless/api/documents/41/download/": () => + new Response( + new ReadableStream({ + start(controller) { + controller.enqueue(new Uint8Array(800)); + controller.enqueue(new Uint8Array(800)); + controller.close(); + }, + }), + ), + }); + expect( + await failure(sourceClient(PAPERLESS).download("41", 1024)), + ).toBeInstanceOf(DocumentSourceTooLargeError); + }); +}); + +describe("Papra", () => { + const tags = { + tags: [ + { id: "tag_b", name: "Lab results" }, + { id: "tag_a", name: "Health" }, + ], + }; + + it("sends the name and the tag through searchQuery and re-checks both", async () => { + serve({ + "/api/organizations/org_123/tags": () => json(tags), + "/api/organizations/org_123/documents": () => + json({ + documentsCount: 60, + documents: [ + { + id: "doc_1", + name: "Blood test.pdf", + originalName: "Blood test.pdf", + originalSize: 1234, + createdAt: "2025-03-04T09:00:00.000Z", + tags: [{ id: "tag_b", name: "Lab results" }], + }, + { + id: "doc_2", + name: "Other.pdf", + createdAt: "2025-03-04T09:00:00.000Z", + tags: [{ id: "tag_a", name: "Health" }], + }, + { + id: "doc_3", + name: "Too old.pdf", + documentDate: "2018-01-01", + createdAt: "2025-03-04T09:00:00.000Z", + tags: [{ id: "tag_b", name: "Lab results" }], + }, + ], + }), + }); + const result = await sourceClient(PAPRA).search({ + q: "blood", + tagId: "tag_b", + from: "2025-01-01", + to: null, + page: 1, + }); + const list = calledUrls().find((u) => u.pathname.endsWith("/documents")); + expect(list?.searchParams.get("searchQuery")).toBe( + 'blood tag:"Lab results"', + ); + expect(list?.searchParams.get("pageIndex")).toBe("0"); + expect(list?.searchParams.get("pageSize")).toBe("25"); + expect(safeFetch.mock.calls[0][1].headers.Authorization).toBe( + "Bearer ppk_secret", + ); + expect(result).toEqual({ + items: [ + { + sourceId: "doc_1", + title: "Blood test", + date: "2025-03-04", + tags: ["Lab results"], + sizeBytes: 1234, + }, + ], + hasMore: true, + }); + }); + + it("matches nothing for a tag that no longer exists", async () => { + serve({ "/api/organizations/org_123/tags": () => json(tags) }); + await expect( + sourceClient(PAPRA).search({ + q: "", + tagId: "tag_gone", + from: null, + to: null, + page: 1, + }), + ).resolves.toEqual({ items: [], hasMore: false }); + expect(calledUrls().some((u) => u.pathname.endsWith("/documents"))).toBe( + false, + ); + }); + + it("lists tags by name and maps an unknown organization to notFound", async () => { + serve({ "/api/organizations/org_123/tags": () => json(tags) }); + await expect(sourceClient(PAPRA).tags()).resolves.toEqual([ + { id: "tag_a", name: "Health" }, + { id: "tag_b", name: "Lab results" }, + ]); + + serve({}); + const err = await failure(sourceClient(PAPRA).test()); + expect((err as DocumentSourceError).code).toBe("notFound"); + }); + + it("reads a document and downloads its file", async () => { + serve({ + "/api/organizations/org_123/documents/doc_1/file": () => + new Response(new Uint8Array([1, 2, 3])), + "/api/organizations/org_123/documents/doc_1": () => + json({ + document: { + id: "doc_1", + name: "Blood test.pdf", + originalName: "blood.pdf", + documentDate: "2025-02-01", + createdAt: "2025-03-04T09:00:00.000Z", + }, + }), + }); + const client = sourceClient(PAPRA); + await expect(client.document("doc_1")).resolves.toEqual({ + title: "Blood test", + date: "2025-02-01", + filename: "blood.pdf", + }); + const file = await client.download("doc_1", 1024); + expect([...file.bytes]).toEqual([1, 2, 3]); + }); +}); diff --git a/src/lib/documents/sources/__tests__/origins.test.ts b/src/lib/documents/sources/__tests__/origins.test.ts new file mode 100644 index 000000000..1dace82ef --- /dev/null +++ b/src/lib/documents/sources/__tests__/origins.test.ts @@ -0,0 +1,110 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; + +import { + configuredDocumentSourceOrigins, + documentSourcesAvailable, + evaluateSourceBaseUrl, + listedDocumentSourceOrigins, +} from "../origins"; + +const ORIGINAL = process.env.DOCUMENT_SOURCE_ORIGINS; + +afterEach(() => { + if (ORIGINAL === undefined) delete process.env.DOCUMENT_SOURCE_ORIGINS; + else process.env.DOCUMENT_SOURCE_ORIGINS = ORIGINAL; + vi.restoreAllMocks(); +}); + +describe("DOCUMENT_SOURCE_ORIGINS", () => { + it("is off when unset or empty", () => { + delete process.env.DOCUMENT_SOURCE_ORIGINS; + expect(documentSourcesAvailable()).toBe(false); + process.env.DOCUMENT_SOURCE_ORIGINS = " "; + expect(documentSourcesAvailable()).toBe(false); + }); + + it("keeps valid exact origins and drops malformed ones with a redacted warning", () => { + const warn = vi.spyOn(console, "warn").mockImplementation(() => {}); + process.env.DOCUMENT_SOURCE_ORIGINS = + "https://paperless.example.com, http://papra.lan:1221, https://user:secret@bad.example/x?token=abc, http://169.254.169.254"; + const origins = configuredDocumentSourceOrigins(); + expect([...origins].sort()).toEqual([ + "http://papra.lan:1221", + "https://paperless.example.com", + ]); + expect(documentSourcesAvailable()).toBe(true); + const logged = warn.mock.calls.map((c) => String(c[0])).join("\n"); + expect(logged).toContain("DOCUMENT_SOURCE_ORIGINS"); + expect(logged).not.toContain("secret"); + expect(logged).not.toContain("token=abc"); + }); + + it("lists the origins for the settings card", () => { + process.env.DOCUMENT_SOURCE_ORIGINS = + "https://paperless.example.com,http://papra.lan:1221"; + expect(listedDocumentSourceOrigins()).toEqual([ + "http://papra.lan:1221", + "https://paperless.example.com", + ]); + }); +}); + +describe("evaluateSourceBaseUrl", () => { + const listed = new Set(["https://paperless.example.com", "http://10.0.0.5"]); + + it("accepts a listed origin and keeps a base path without the trailing slash", () => { + expect( + evaluateSourceBaseUrl("https://paperless.example.com/paperless/", listed), + ).toEqual({ + ok: true, + origin: "https://paperless.example.com", + baseUrl: "https://paperless.example.com/paperless", + }); + expect(evaluateSourceBaseUrl("http://10.0.0.5", listed)).toEqual({ + ok: true, + origin: "http://10.0.0.5", + baseUrl: "http://10.0.0.5", + }); + }); + + it("refuses an origin that is not listed, public or private", () => { + for (const url of [ + "https://example.org", + "http://10.0.0.6", + "http://192.168.1.10:8000", + "http://169.254.169.254", + "http://localhost:8000", + ]) { + expect(evaluateSourceBaseUrl(url, listed), url).toEqual({ + ok: false, + reason: "notAllowed", + }); + } + }); + + it("does not turn a listed host into a suffix, sibling port or other scheme grant", () => { + for (const url of [ + "https://evil.paperless.example.com", + "https://paperless.example.com:8443", + "http://paperless.example.com", + "https://paperless.example.com.evil.test", + ]) { + expect(evaluateSourceBaseUrl(url, listed).ok, url).toBe(false); + } + }); + + it("refuses userinfo, a query, a fragment and non-http schemes", () => { + for (const url of [ + "https://user:pass@paperless.example.com", + "https://paperless.example.com/?token=x", + "https://paperless.example.com/#x", + "ftp://paperless.example.com", + "not a url", + ]) { + expect(evaluateSourceBaseUrl(url, listed), url).toEqual({ + ok: false, + reason: "invalid", + }); + } + }); +}); From c840d1012ace178c8abb3a51edc3a6e6414395b7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Sat, 26 Sep 2026 13:07:44 +0200 Subject: [PATCH 04/18] test(documents): exercise the picker's routes against the database with the source faked at the fetch layer --- tests/integration/document-sources.test.ts | 585 +++++++++++++++++++++ 1 file changed, 585 insertions(+) create mode 100644 tests/integration/document-sources.test.ts diff --git a/tests/integration/document-sources.test.ts b/tests/integration/document-sources.test.ts new file mode 100644 index 000000000..e729e7aed --- /dev/null +++ b/tests/integration/document-sources.test.ts @@ -0,0 +1,585 @@ +/** + * The document picker (#1038) against a real Postgres, the real auth resolver + * and the real routes. Only the network is replaced: `safeFetch` answers from + * an in-memory Paperless-ngx, so every assertion about what was fetched (and + * what was not) is a count on that fake. + * + * 1. With `DOCUMENT_SOURCE_ORIGINS` unset the picker is off. + * 2. Saving refuses an unlisted origin and a refused token, stores nothing + * then, and otherwise stores the token encrypted and never returns it. + * 3. Only the owner's browser session reaches the routes: a `["*"]` Bearer + * and a delegate acting on the owner's record are refused. + * 4. Search reports what the vault holds per source key; import stores + * through the upload's ingest path with provenance; a re-import is a + * duplicate and a document deleted in HealthLog stays deleted, both + * without a download. + * 5. An import links to the record it was picked for, and a foreign record + * is refused before anything is fetched. + * 6. An origin the operator removed cuts the stored connection off. + */ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { NextRequest } from "next/server"; + +process.env.ENCRYPTION_KEY ??= + "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"; +process.env.API_TOKEN_HMAC_KEY ??= + "test-hmac-key-document-sources-32-bytes-min-0987654321abcd"; + +import { cookieJar, headerJar } from "./mock-next-headers"; +import { getPrismaClient, truncateAllTables } from "./setup"; + +vi.mock("next/headers", async () => { + const { cookieJar, headerJar } = await import("./mock-next-headers"); + return { + headers: vi.fn(async () => ({ + get: (name: string) => headerJar.get(name.toLowerCase()) ?? null, + })), + cookies: vi.fn(async () => ({ + get: (name: string) => { + const value = cookieJar.get(name); + return value ? { name, value } : undefined; + }, + set: (name: string, value: string) => { + cookieJar.set(name, value); + }, + delete: (name: string) => { + cookieJar.delete(name); + }, + })), + }; +}); + +vi.mock("@/lib/db-compat", () => ({ + ensureDbCompatibility: vi.fn().mockResolvedValue(undefined), +})); + +// ── The fake Paperless-ngx behind `safeFetch` ────────────────────────────── + +const ORIGIN = "http://paperless.lan:8000"; +const TOKEN = "paperless-token-0123456789-secret"; + +const PDF = (tag: string) => + Buffer.from(`%PDF-1.7\n% ${tag}\n1 0 obj\n<<>>\nendobj\n%%EOF\n`); + +const DOCS = [ + { + id: 41, + title: "Blood test", + created: "2024-03-04", + tags: [7], + original_file_name: "blood.pdf", + bytes: PDF("blood"), + }, + { + id: 42, + title: "Discharge letter", + created: "2023-11-20", + tags: [], + original_file_name: "discharge.pdf", + bytes: PDF("discharge"), + }, +]; + +const fake = vi.hoisted(() => ({ + calls: [] as Array<{ url: string; opts: Record }>, + acceptToken: "", +})); + +vi.mock("@/lib/safe-fetch", async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + safeFetch: vi.fn( + async ( + target: string, + init: RequestInit, + opts: Record, + ) => { + fake.calls.push({ url: String(target), opts }); + const url = new URL(String(target)); + if (url.origin !== opts.operatorApprovedPrivateOrigin) { + throw new actual.SafeFetchError("not approved", "private_host"); + } + const auth = (init.headers as Record).Authorization; + if (auth !== `Token ${fake.acceptToken}`) { + return new Response("{}", { status: 401 }); + } + const json = (body: unknown) => + new Response(JSON.stringify(body), { + headers: { "content-type": "application/json" }, + }); + const path = url.pathname; + if (path === "/api/tags/") { + return json({ results: [{ id: 7, name: "Health" }], next: null }); + } + if (path === "/api/documents/") { + const q = ( + url.searchParams.get("title__icontains") ?? "" + ).toLowerCase(); + return json({ + next: null, + results: DOCS.filter((d) => d.title.toLowerCase().includes(q)).map( + ({ bytes: _bytes, ...doc }) => doc, + ), + }); + } + const download = /^\/api\/documents\/(\d+)\/download\/$/.exec(path); + if (download) { + const doc = DOCS.find((d) => String(d.id) === download[1]); + return doc + ? new Response(new Uint8Array(doc.bytes)) + : new Response("", { status: 404 }); + } + const detail = /^\/api\/documents\/(\d+)\/$/.exec(path); + if (detail) { + const doc = DOCS.find((d) => String(d.id) === detail[1]); + if (!doc) return new Response("{}", { status: 404 }); + const { bytes: _bytes, ...meta } = doc; + return json(meta); + } + return new Response("{}", { status: 404 }); + }, + ), + }; +}); + +import { decrypt } from "@/lib/crypto"; +import { hashToken } from "@/lib/auth/hmac"; +import { acceptGrant, inviteGrant } from "@/lib/sharing/grants"; + +// ── Helpers ──────────────────────────────────────────────────────────────── + +let userId = ""; +let sessionId = ""; + +async function makeUser(name: string) { + return getPrismaClient().user.create({ + data: { + username: name, + email: `${name}@example.test`, + role: "USER", + timezone: "UTC", + modulePreferencesJson: { inboundDocuments: true }, + }, + }); +} + +async function signIn(id: string) { + return getPrismaClient().session.create({ + data: { userId: id, expiresAt: new Date(Date.now() + 600_000) }, + }); +} + +async function seedOwner() { + const user = await makeUser("owner"); + const session = await signIn(user.id); + userId = user.id; + sessionId = session.id; + asCookie(sessionId); +} + +function asCookie(id: string) { + headerJar.clear(); + cookieJar.set("healthlog_session", id); +} + +type Params = { params: Promise<{ system: string }> }; +const paperless: Params = { + params: Promise.resolve({ system: "paperless" }), +}; +type Handler = (r: NextRequest, c: Params) => Promise; + +function jsonRequest(url: string, method: string, body?: unknown) { + const headers: Record = {}; + const auth = headerJar.get("authorization"); + if (auth) headers.authorization = auth; + if (body !== undefined) headers["content-type"] = "application/json"; + return new NextRequest(`https://health.example${url}`, { + method, + headers, + body: body === undefined ? undefined : JSON.stringify(body), + } as never); +} + +async function status(): Promise { + const { GET } = await import("@/app/api/documents/sources/route"); + return (GET as unknown as (r: NextRequest) => Promise)( + jsonRequest("/api/documents/sources", "GET"), + ); +} + +async function save(body: unknown): Promise { + const { PUT } = await import("@/app/api/documents/sources/[system]/route"); + return (PUT as unknown as Handler)( + jsonRequest("/api/documents/sources/paperless", "PUT", body), + paperless, + ); +} + +async function disconnect(): Promise { + const { DELETE } = await import("@/app/api/documents/sources/[system]/route"); + return (DELETE as unknown as Handler)( + jsonRequest("/api/documents/sources/paperless", "DELETE"), + paperless, + ); +} + +async function search(query = ""): Promise { + const { GET } = + await import("@/app/api/documents/sources/[system]/search/route"); + return (GET as unknown as Handler)( + jsonRequest(`/api/documents/sources/paperless/search?${query}`, "GET"), + paperless, + ); +} + +async function importDoc(body: unknown): Promise { + const { POST } = + await import("@/app/api/documents/sources/[system]/import/route"); + return (POST as unknown as Handler)( + jsonRequest("/api/documents/sources/paperless/import", "POST", body), + paperless, + ); +} + +async function connect() { + fake.acceptToken = TOKEN; + const res = await save({ baseUrl: `${ORIGIN}/`, token: TOKEN }); + expect(res.status).toBe(200); + return res; +} + +function downloads(): number { + return fake.calls.filter((c) => c.url.includes("/download/")).length; +} + +beforeEach(async () => { + await truncateAllTables(getPrismaClient()); + cookieJar.clear(); + headerJar.clear(); + fake.calls.length = 0; + fake.acceptToken = TOKEN; + process.env.DOCUMENT_SOURCE_ORIGINS = ORIGIN; +}); + +afterEach(() => { + delete process.env.DOCUMENT_SOURCE_ORIGINS; +}); + +// ── Tests ────────────────────────────────────────────────────────────────── + +describe("the operator's list switches the picker on", () => { + it("is off without DOCUMENT_SOURCE_ORIGINS, and nothing is dialled", async () => { + delete process.env.DOCUMENT_SOURCE_ORIGINS; + await seedOwner(); + const off = await status(); + expect(off.status).toBe(200); + expect((await off.json()).data).toEqual({ + available: false, + allowedOrigins: [], + connections: [], + }); + const refused = await save({ baseUrl: ORIGIN, token: TOKEN }); + expect(refused.status).toBe(404); + expect((await refused.json()).meta.errorCode).toBe( + "documents.sources.unavailable", + ); + expect(fake.calls).toHaveLength(0); + }); +}); + +describe("saving a connection", () => { + it("refuses an origin the operator did not list, public or private, without dialling", async () => { + await seedOwner(); + for (const baseUrl of [ + "http://10.0.0.9:8000", + "https://paperless.example.org", + "http://paperless.lan:8001", + ]) { + const res = await save({ baseUrl, token: TOKEN }); + expect(res.status, baseUrl).toBe(422); + expect((await res.json()).meta.errorCode).toBe( + "documents.sources.originNotAllowed", + ); + } + expect(fake.calls).toHaveLength(0); + expect(await getPrismaClient().documentSourceConnection.count()).toBe(0); + }); + + it("refuses a token the source rejects and stores nothing", async () => { + await seedOwner(); + fake.acceptToken = "the-real-one"; + const res = await save({ baseUrl: ORIGIN, token: TOKEN }); + expect(res.status).toBe(502); + expect((await res.json()).meta).toMatchObject({ + errorCode: "documents.sources.authRefused", + upstreamStatus: 401, + }); + expect(await getPrismaClient().documentSourceConnection.count()).toBe(0); + }); + + it("stores the token encrypted and never returns it", async () => { + await seedOwner(); + const saved = await connect(); + const savedText = await saved.text(); + expect(savedText).not.toContain(TOKEN); + expect(JSON.parse(savedText).data).toMatchObject({ + system: "PAPERLESS", + baseUrl: ORIGIN, + hasToken: true, + originAllowed: true, + }); + + const row = + await getPrismaClient().documentSourceConnection.findFirstOrThrow(); + expect(row.userId).toBe(userId); + expect(row.tokenEncrypted).not.toContain(TOKEN); + expect(decrypt(row.tokenEncrypted)).toBe(TOKEN); + + const read = await (await status()).text(); + expect(read).not.toContain(TOKEN); + expect(JSON.parse(read).data.allowedOrigins).toEqual([ORIGIN]); + + // A save without a token keeps the stored one. + const again = await save({ baseUrl: ORIGIN }); + expect(again.status).toBe(200); + const kept = + await getPrismaClient().documentSourceConnection.findFirstOrThrow(); + expect(decrypt(kept.tokenEncrypted)).toBe(TOKEN); + + expect((await disconnect()).status).toBe(200); + expect(await getPrismaClient().documentSourceConnection.count()).toBe(0); + }); +}); + +describe("only the owner's browser session", () => { + it("refuses a wildcard Bearer token", async () => { + await seedOwner(); + await connect(); + const raw = "hlk_document-sources-wildcard-token-000"; + await getPrismaClient().apiToken.create({ + data: { + userId, + name: "native", + tokenHash: hashToken(raw), + permissions: ["*"], + }, + }); + cookieJar.clear(); + headerJar.set("authorization", `Bearer ${raw}`); + fake.calls.length = 0; + for (const res of [ + await status(), + await search("q=blood"), + await importDoc({ sourceId: "41" }), + ]) { + expect(res.status).toBe(403); + expect((await res.json()).meta.errorCode).toBe( + "documents.sources.browserOnly", + ); + } + expect(fake.calls).toHaveLength(0); + }); + + it("refuses a delegate acting on the owner's record", async () => { + await seedOwner(); + await connect(); + const delegate = await makeUser("delegate"); + const invited = await inviteGrant({ + grantorId: userId, + granteeId: delegate.id, + access: "WRITE", + scope: null, + }); + await acceptGrant({ grantId: invited.id, granteeId: delegate.id }); + const session = await signIn(delegate.id); + const switched = await getPrismaClient().session.update({ + where: { id: session.id }, + data: { actingAsUserId: userId }, + }); + asCookie(session.id); + headerJar.set("x-healthlog-record-epoch", String(switched.recordEpoch)); + headerJar.set("x-healthlog-record-scope", userId); + fake.calls.length = 0; + + for (const res of [ + await status(), + await search("q=blood"), + await importDoc({ sourceId: "41" }), + ]) { + expect(res.status).toBe(403); + } + expect(fake.calls).toHaveLength(0); + expect(await getPrismaClient().inboundDocument.count()).toBe(0); + }); +}); + +describe("search and import", () => { + it("imports through the ingest path; a re-import is a duplicate and a deleted one stays deleted", async () => { + await seedOwner(); + await connect(); + const prisma = getPrismaClient(); + + const first = await search("q=blood"); + expect(first.status).toBe(200); + expect((await first.json()).data).toEqual({ + page: 1, + hasMore: false, + results: [ + { + sourceId: "41", + title: "Blood test", + date: "2024-03-04", + tags: ["Health"], + sizeBytes: null, + state: "new", + documentId: null, + }, + ], + }); + + const imported = await importDoc({ sourceId: "41" }); + expect(imported.status).toBe(201); + const receipt = (await imported.json()).data; + expect(receipt).toMatchObject({ outcome: "imported", linked: false }); + const stored = await prisma.inboundDocument.findUniqueOrThrow({ + where: { id: receipt.documentId }, + omit: { contentEncrypted: true }, + }); + expect(stored).toMatchObject({ + userId, + title: "Blood test", + filename: "blood.pdf", + sourceSystem: "PAPERLESS", + sourceId: "41", + mimeType: "application/pdf", + aiReadDeferred: false, + kind: "OTHER", + }); + expect(stored.documentDate?.toISOString().slice(0, 10)).toBe("2024-03-04"); + expect(downloads()).toBe(1); + + const listed = (await (await search("q=blood")).json()).data.results[0]; + expect(listed).toMatchObject({ + state: "imported", + documentId: receipt.documentId, + }); + + const again = await importDoc({ sourceId: "41" }); + expect(again.status).toBe(200); + expect((await again.json()).data).toEqual({ + outcome: "duplicate", + documentId: receipt.documentId, + linked: false, + }); + expect(downloads()).toBe(1); + + await prisma.inboundDocument.update({ + where: { id: receipt.documentId }, + data: { deletedAt: new Date() }, + }); + expect( + (await (await search("q=blood")).json()).data.results[0], + ).toMatchObject({ state: "deleted", documentId: null }); + const revived = await importDoc({ sourceId: "41" }); + expect(revived.status).toBe(200); + expect((await revived.json()).data.outcome).toBe("deleted"); + expect(downloads()).toBe(1); + expect(await prisma.inboundDocument.count()).toBe(1); + }); + + it("answers a file already uploaded by hand as a duplicate and remembers the key", async () => { + await seedOwner(); + await connect(); + const prisma = getPrismaClient(); + + // The same bytes, uploaded through the vault without a source key. + const form = new FormData(); + form.append("file", new Blob([new Uint8Array(DOCS[1].bytes)]), "x.pdf"); + const { POST } = await import("@/app/api/documents/inbound/route"); + const uploaded = await ( + POST as unknown as (r: Request) => Promise + )( + new Request("http://localhost/api/documents/inbound", { + method: "POST", + body: form, + }), + ); + expect(uploaded.status).toBe(201); + const uploadedId = (await uploaded.json()).data.id; + + const picked = await importDoc({ sourceId: "42" }); + expect(picked.status).toBe(200); + expect((await picked.json()).data).toMatchObject({ + outcome: "duplicate", + documentId: uploadedId, + }); + expect(await prisma.inboundDocument.count()).toBe(1); + expect( + await prisma.documentSourceAlias.count({ + where: { + documentId: uploadedId, + sourceSystem: "PAPERLESS", + sourceId: "42", + }, + }), + ).toBe(1); + }); + + it("links to the visit it was picked for, and refuses someone else's before fetching", async () => { + await seedOwner(); + await connect(); + const prisma = getPrismaClient(); + const visit = await prisma.encounter.create({ + data: { userId, occurredAt: new Date("2024-03-04T10:00:00Z") }, + }); + const stranger = await makeUser("stranger"); + const foreign = await prisma.encounter.create({ + data: { userId: stranger.id, occurredAt: new Date() }, + }); + + const refused = await importDoc({ + sourceId: "41", + link: { kind: "encounter", id: foreign.id }, + }); + expect(refused.status).toBe(404); + expect((await refused.json()).meta.errorCode).toBe( + "documents.sources.linkTargetNotFound", + ); + expect(downloads()).toBe(0); + + const linked = await importDoc({ + sourceId: "41", + link: { kind: "encounter", id: visit.id }, + }); + expect(linked.status).toBe(201); + const data = (await linked.json()).data; + expect(data.linked).toBe(true); + expect( + await prisma.encounterDocumentLink.count({ + where: { encounterId: visit.id, documentId: data.documentId }, + }), + ).toBe(1); + }); + + it("cuts a stored connection off once the operator removes its origin", async () => { + await seedOwner(); + await connect(); + fake.calls.length = 0; + process.env.DOCUMENT_SOURCE_ORIGINS = "https://other.example.com"; + + const res = await search("q=blood"); + expect(res.status).toBe(422); + expect((await res.json()).meta.errorCode).toBe( + "documents.sources.originNotAllowed", + ); + const imported = await importDoc({ sourceId: "41" }); + expect(imported.status).toBe(422); + expect(fake.calls).toHaveLength(0); + + const read = (await (await status()).json()).data; + expect(read.connections[0]).toMatchObject({ originAllowed: false }); + // Still removable. + expect((await disconnect()).status).toBe(200); + }); +}); From caf3eb323d3a618ebd73d6b75c51abfb7009adc9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Sat, 26 Sep 2026 13:13:51 +0200 Subject: [PATCH 05/18] docs(api): publish the document picker routes as cookie-only, and freeze them as owner-only --- docs/api/openapi.yaml | 668 ++++++++++++++++++ .../document-sources-owner-only-guard.test.ts | 106 +++ src/lib/openapi/routes/document-sources.ts | 402 +++++++++++ src/lib/openapi/routes/index.ts | 5 + 4 files changed, 1181 insertions(+) create mode 100644 src/__tests__/document-sources-owner-only-guard.test.ts create mode 100644 src/lib/openapi/routes/document-sources.ts diff --git a/docs/api/openapi.yaml b/docs/api/openapi.yaml index 89ecc1799..0827581f2 100644 --- a/docs/api/openapi.yaml +++ b/docs/api/openapi.yaml @@ -20966,6 +20966,365 @@ paths: application/json: schema: $ref: "#/components/schemas/ErrorEnvelope" + /api/documents/sources: + get: + tags: + - Documents + summary: "Document archives: availability and connections" + description: "Whether the operator enabled the document picker (`available`, true when `DOCUMENT_SOURCE_ORIGINS` lists + at least one origin), the listed origins, and the caller's own saved connections to Paperless-ngx or Papra. A + connection never carries its token: `hasToken` is always true for a saved one, and no route returns the token in + whole or in part. `originAllowed` is false once the operator removed the connection's origin from the list. No + request leaves the server. Cookie session only; not delegable." + security: &a52 + - cookieAuth: [] + responses: + "200": + description: The picker's state for the caller. + content: + application/json: + schema: + $ref: "#/components/schemas/DocumentSourcesStatusEnvelope" + "401": *a1 + "403": &a53 + description: "Refused before any request to the source: `documents.sources.browserOnly` for a Bearer caller, + `sharing.not_permitted` while acting on another person's record (a delegate or guardian never uses the + owner's archive), or `module.disabled` when the Documents module is off." + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorEnvelope" + "422": *a4 + "429": *a5 + /api/documents/sources/{system}: + put: + tags: + - Documents + summary: Save a connection to a document archive + description: "Saves or replaces the caller's connection to Paperless-ngx (`paperless`: base address and API token) or + Papra (`papra`: base address, organization id, API key with `documents:read` and `tags:read`). The base address + may carry a path; its origin must be listed in `DOCUMENT_SOURCE_ORIGINS`. The connection is tested live before + anything is stored. Leaving out `token` keeps the stored one (the first save must carry it). The token is stored + encrypted (AES-256-GCM) and never returned. Ten saves and tests a minute per person. Cookie session only; not + delegable." + security: *a52 + parameters: + - &a54 + name: system + in: path + required: true + description: "`paperless` (Paperless-ngx) or `papra` (Papra)." + schema: + type: string + enum: + - paperless + - papra + requestBody: + required: true + content: + application/json: + schema: + type: object + properties: + baseUrl: + type: string + minLength: 1 + maxLength: 2048 + organizationId: + type: string + minLength: 1 + maxLength: 128 + pattern: ^[\x21-\x7e]+$ + token: + type: string + minLength: 1 + maxLength: 512 + pattern: ^[\x21-\x7e]+$ + required: + - baseUrl + responses: + "200": + description: Saved and verified. The connection, without its token. + content: + application/json: + schema: + $ref: "#/components/schemas/DocumentSourceConnectionEnvelope" + "401": *a1 + "403": *a53 + "404": &a55 + description: "`documents.sources.unavailable`: the operator has not listed any origin in `DOCUMENT_SOURCE_ORIGINS`, so + the picker is off. `documents.sources.notConnected`: no connection is saved for this system. + `documents.sources.notFound`: the source has no such document, or (Papra) no such organization." + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorEnvelope" + "422": + description: "`documents.sources.originNotAllowed`, `documents.sources.invalidAddress`, + `documents.sources.organizationRequired` (Papra), `documents.sources.tokenRequired` (first save without a + token), or body validation." + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorEnvelope" + "429": *a5 + "502": &a56 + description: "The source could not answer usefully, with the reason in `meta.errorCode` and, when the source answered at + all, its HTTP status in `meta.upstreamStatus` (its body is never passed on): `documents.sources.unreachable` + (no connection, timeout, or a destination refused at dial time), `documents.sources.redirected` (the source + answered with a redirect, which is never followed), `documents.sources.authRefused` (401/403 from the + source), `documents.sources.versionTooOld` (Paperless-ngx older than 2.16 / API version 9), + `documents.sources.badResponse` (not the documented JSON, or larger than 1 MiB)." + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorEnvelope" + delete: + tags: + - Documents + summary: Remove a connection to a document archive + description: Deletes the caller's connection for this system, and its token with it. Works even after the operator + removed the origin from the list. `disconnected` is false when there was nothing to delete. Cookie session only; + not delegable. + security: *a52 + parameters: + - *a54 + responses: + "200": + description: Removed, or there was nothing to remove. + content: + application/json: + schema: + $ref: "#/components/schemas/DocumentSourceDisconnectEnvelope" + "401": *a1 + "403": *a53 + "422": *a4 + "429": *a5 + /api/documents/sources/{system}/test: + post: + tags: + - Documents + summary: Test a saved connection + description: One request to the source with the stored token. Answers `ok` with the round-trip time and stamps + `lastVerifiedAt`, or the reason in `meta.errorCode`. Shares the save's allowance of ten a minute. Cookie session + only; not delegable. + security: *a52 + parameters: + - *a54 + responses: + "200": + description: The source answered. + content: + application/json: + schema: + $ref: "#/components/schemas/DocumentSourceTestEnvelope" + "401": *a1 + "403": *a53 + "404": *a55 + "422": &a57 + description: "`documents.sources.originNotAllowed`: the base address's origin is not on the operator's list (checked on + every request, so a connection saved earlier is cut off once its origin is removed), or another validation + failure." + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorEnvelope" + "429": *a5 + "502": *a56 + /api/documents/sources/{system}/tags: + get: + tags: + - Documents + summary: Tags of a connected archive + description: The source's tags for the picker's filter, the first 250 by name, in the source's own ids. Shares the + search allowance of sixty a minute. Cookie session only; not delegable. + security: *a52 + parameters: + - *a54 + responses: + "200": + description: The tags. + content: + application/json: + schema: + $ref: "#/components/schemas/DocumentSourceTagsEnvelope" + "401": *a1 + "403": *a53 + "404": *a55 + "422": *a57 + "429": *a5 + "502": *a56 + /api/documents/sources/{system}/search: + get: + tags: + - Documents + summary: Search a connected archive by name, tag and date + description: "Searches the source 25 documents at a time. `q` matches the document's name (Paperless-ngx title, Papra + name); `tag` narrows to one of the source's tags by id; `from` / `to` to a date range, inclusive (Papra's range + is applied to each page after it arrives, so a page can hold fewer than 25). Each result carries what the vault + already holds under that source key: `new`, `imported` with the vault document's `documentId`, or `deleted` + (deleted in HealthLog; importing it again stores nothing). Nothing is stored. Sixty searches a minute, shared + with the tag read. Cookie session only; not delegable." + security: *a52 + parameters: + - *a54 + - name: q + in: query + required: false + schema: + type: string + maxLength: 200 + - name: tag + in: query + required: false + schema: + type: string + maxLength: 64 + - name: from + in: query + required: false + schema: + type: string + format: date + - name: to + in: query + required: false + schema: + type: string + format: date + - name: page + in: query + required: false + schema: + type: integer + minimum: 1 + maximum: 500 + default: 1 + responses: + "200": + description: One page of results. + content: + application/json: + schema: + $ref: "#/components/schemas/DocumentSourceSearchEnvelope" + "401": *a1 + "403": *a53 + "404": *a55 + "422": *a57 + "429": *a5 + "502": *a56 + /api/documents/sources/{system}/import: + post: + tags: + - Documents + summary: Import one picked document + description: "Fetches one document from the source and stores it in the vault through the same path as an upload, with + `sourceSystem` / `sourceId` provenance. A source key the vault already holds is answered without a download: + `duplicate` for a live document (same id), `deleted` for one the person deleted in HealthLog, which is never + stored again. Bytes already in the vault under another key are `duplicate` too, and the key is remembered for + that document. `link` files the document against the record the picker was opened from (a condition, a visit or + a vaccination, which must be the caller's own and live; checked before anything is fetched), including a + duplicate. `kind` sets the document type (default OTHER). Automatic AI reading follows the person's own setting. + Counts against the person's upload allowance (60 an hour) and storage quota. 201 for a new document, 200 + otherwise. Cookie session only; not delegable." + security: *a52 + parameters: + - *a54 + requestBody: + required: true + content: + application/json: + schema: + type: object + properties: + sourceId: + type: string + minLength: 1 + maxLength: 128 + pattern: ^[\x21-\x7e]+$ + kind: + type: string + enum: + - DOCTOR_REPORT + - DISCHARGE_LETTER + - LAB_RESULT + - IMAGING + - PRESCRIPTION + - REFERRAL + - INSURANCE + - VACCINATION + - OTHER + link: + type: object + properties: + kind: + type: string + enum: + - conditionEpisode + - encounter + - vaccination + id: + type: string + minLength: 1 + maxLength: 64 + required: + - kind + - id + required: + - sourceId + responses: + "200": + description: Already held (`duplicate`) or deleted in HealthLog (`deleted`). + content: + application/json: + schema: + $ref: "#/components/schemas/DocumentSourceImportHeldEnvelope" + "201": + description: Imported. + content: + application/json: + schema: + $ref: "#/components/schemas/DocumentSourceImportEnvelope" + "401": *a1 + "403": *a53 + "404": + description: "`documents.sources.unavailable`: the operator has not listed any origin in `DOCUMENT_SOURCE_ORIGINS`, so + the picker is off. `documents.sources.notConnected`: no connection is saved for this system. + `documents.sources.notFound`: the source has no such document, or (Papra) no such organization. + `documents.sources.linkTargetNotFound`: the record in `link` is not the caller's own live conditionEpisode / + encounter / vaccination." + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorEnvelope" + "409": + description: "`documents.inbound.sourceAliasLimit`: these bytes are already stored under too many source ids." + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorEnvelope" + "413": + description: "`documents.inbound.fileTooLarge` (with `maxFileBytes`) or `documents.inbound.quotaExceeded` (with + `quotaBytes` and `usedBytes`)." + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorEnvelope" + "415": + description: "`documents.inbound.fileType`: the file is not a type the vault accepts." + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorEnvelope" + "422": *a57 + "429": + description: "`documents.inbound.rateLimited` (the person's hourly upload allowance, with the rate-limit headers) or + `documents.inbound.uploadBusy` (too many uploads in progress; `Retry-After: 1`)." + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorEnvelope" + "502": *a56 /api/bugreport: post: security: [] @@ -72408,6 +72767,315 @@ components: - data - error additionalProperties: false + DocumentSourcesStatusEnvelope: + type: object + properties: + data: + $ref: "#/components/schemas/DocumentSourcesStatus" + error: + type: "null" + meta: + type: object + properties: + requestId: + type: string + additionalProperties: false + required: + - data + - error + additionalProperties: false + DocumentSourcesStatus: + type: object + properties: + available: + type: boolean + allowedOrigins: + type: array + items: + type: string + connections: + type: array + items: + $ref: "#/components/schemas/DocumentSourceConnection" + required: + - available + - allowedOrigins + - connections + additionalProperties: false + DocumentSourceConnection: + type: object + properties: + system: + type: string + enum: + - PAPERLESS + - PAPRA + baseUrl: + type: string + organizationId: + anyOf: + - type: string + - type: "null" + hasToken: + type: boolean + const: true + lastVerifiedAt: + anyOf: + - type: string + - type: "null" + originAllowed: + type: boolean + required: + - system + - baseUrl + - organizationId + - hasToken + - lastVerifiedAt + - originAllowed + additionalProperties: false + DocumentSourceConnectionEnvelope: + type: object + properties: + data: + $ref: "#/components/schemas/DocumentSourceConnection" + error: + type: "null" + meta: + type: object + properties: + requestId: + type: string + additionalProperties: false + required: + - data + - error + additionalProperties: false + DocumentSourceDisconnectEnvelope: + type: object + properties: + data: + $ref: "#/components/schemas/DocumentSourceDisconnect" + error: + type: "null" + meta: + type: object + properties: + requestId: + type: string + additionalProperties: false + required: + - data + - error + additionalProperties: false + DocumentSourceDisconnect: + type: object + properties: + disconnected: + type: boolean + required: + - disconnected + additionalProperties: false + DocumentSourceTestEnvelope: + type: object + properties: + data: + $ref: "#/components/schemas/DocumentSourceTest" + error: + type: "null" + meta: + type: object + properties: + requestId: + type: string + additionalProperties: false + required: + - data + - error + additionalProperties: false + DocumentSourceTest: + type: object + properties: + ok: + type: boolean + const: true + latencyMs: + type: integer + minimum: -9007199254740991 + maximum: 9007199254740991 + lastVerifiedAt: + type: string + required: + - ok + - latencyMs + - lastVerifiedAt + additionalProperties: false + DocumentSourceTagsEnvelope: + type: object + properties: + data: + $ref: "#/components/schemas/DocumentSourceTags" + error: + type: "null" + meta: + type: object + properties: + requestId: + type: string + additionalProperties: false + required: + - data + - error + additionalProperties: false + DocumentSourceTags: + type: object + properties: + tags: + type: array + items: + $ref: "#/components/schemas/DocumentSourceTag" + required: + - tags + additionalProperties: false + DocumentSourceTag: + type: object + properties: + id: + type: string + name: + type: string + required: + - id + - name + additionalProperties: false + DocumentSourceSearchEnvelope: + type: object + properties: + data: + $ref: "#/components/schemas/DocumentSourceSearch" + error: + type: "null" + meta: + type: object + properties: + requestId: + type: string + additionalProperties: false + required: + - data + - error + additionalProperties: false + DocumentSourceSearch: + type: object + properties: + results: + type: array + items: + $ref: "#/components/schemas/DocumentSourceResult" + page: + type: integer + minimum: -9007199254740991 + maximum: 9007199254740991 + hasMore: + type: boolean + required: + - results + - page + - hasMore + additionalProperties: false + DocumentSourceResult: + type: object + properties: + sourceId: + type: string + title: + type: string + date: + anyOf: + - type: string + - type: "null" + tags: + type: array + items: + type: string + sizeBytes: + anyOf: + - type: integer + minimum: -9007199254740991 + maximum: 9007199254740991 + - type: "null" + state: + type: string + enum: + - new + - imported + - deleted + documentId: + anyOf: + - type: string + - type: "null" + required: + - sourceId + - title + - date + - tags + - sizeBytes + - state + - documentId + additionalProperties: false + DocumentSourceImportHeldEnvelope: + type: object + properties: + data: + $ref: "#/components/schemas/DocumentSourceImport" + error: + type: "null" + meta: + type: object + properties: + requestId: + type: string + additionalProperties: false + required: + - data + - error + additionalProperties: false + DocumentSourceImport: + type: object + properties: + outcome: + type: string + enum: + - imported + - duplicate + - deleted + documentId: + anyOf: + - type: string + - type: "null" + linked: + type: boolean + required: + - outcome + - documentId + - linked + additionalProperties: false + DocumentSourceImportEnvelope: + type: object + properties: + data: + $ref: "#/components/schemas/DocumentSourceImport" + error: + type: "null" + meta: + type: object + properties: + requestId: + type: string + additionalProperties: false + required: + - data + - error + additionalProperties: false __schema0: anyOf: - type: string diff --git a/src/__tests__/document-sources-owner-only-guard.test.ts b/src/__tests__/document-sources-owner-only-guard.test.ts new file mode 100644 index 000000000..a61741026 --- /dev/null +++ b/src/__tests__/document-sources-owner-only-guard.test.ts @@ -0,0 +1,106 @@ +/** + * The document picker's routes (#1038) are owner-only, cookie-only, and stay + * that way. + * + * A connection is the owner's credential to a system outside HealthLog. A + * delegate or guardian driving it would read the owner's whole Paperless-ngx or + * Papra archive, which no grant covers; a Bearer (the native client, a script) + * has no business managing it either. The rule is in two halves, and this file + * freezes both at every route module under `src/app/api/documents/sources`: + * + * 1. the module calls a bare `requireAuth()` itself (no scope, so a narrow + * token is refused, and a request acting on another record is refused + * 403 `sharing.not_permitted`), and names none of the resolvers that + * would admit a delegate, a guardian, or an actor surface; + * 2. every handler passes the result through `admitDocumentSourceCaller`, + * which refuses any transport but the cookie. + * + * The scan asserts it found the modules it expects, so an empty directory or a + * moved tree fails rather than passing on nothing. + * + * Mutation checks: swap one route's `requireAuth()` for + * `requireRecordAuth("write", "documents")`, or drop its + * `admitDocumentSourceCaller` call, and this file fails naming that module. + */ +import { readFileSync, readdirSync } from "node:fs"; +import { join, relative } from "node:path"; + +import { describe, expect, it } from "vitest"; + +import { stripComments } from "./helpers/source-files"; + +const ROOT = join(process.cwd(), "src/app/api/documents/sources"); + +function routeModules(): string[] { + const out: string[] = []; + const walk = (dir: string) => { + for (const entry of readdirSync(dir, { withFileTypes: true })) { + const full = join(dir, entry.name); + if (entry.isDirectory()) walk(full); + else if (entry.name === "route.ts") out.push(full); + } + }; + walk(ROOT); + return out.map((file) => relative(ROOT, file)).sort(); +} + +const EXPECTED = [ + "[system]/import/route.ts", + "[system]/route.ts", + "[system]/search/route.ts", + "[system]/tags/route.ts", + "[system]/test/route.ts", + "route.ts", +]; + +describe("document-source routes are owner-only and cookie-only", () => { + const modules = routeModules(); + + it("finds exactly the route modules it freezes", () => { + expect(modules).toEqual(EXPECTED); + }); + + it.each(EXPECTED)( + "%s resolves a bare requireAuth and no wider resolver", + (rel) => { + const source = stripComments(readFileSync(join(ROOT, rel), "utf8")); + expect(source).toMatch(/requireAuth\(\s*\)/); + expect(source).not.toMatch(/requireAuth\(\s*[^)\s]/); + for (const wider of [ + "requireRecordAuth", + "requireGuardianAuth", + "requireActorAuth", + "requireCookieAuth", + "requireBearerAuth", + ]) { + expect(source, `${rel} names ${wider}`).not.toContain(wider); + } + }, + ); + + it.each(EXPECTED)( + "%s admits every handler through the cookie check", + (rel) => { + const source = stripComments(readFileSync(join(ROOT, rel), "utf8")); + const handlers = source.match( + /export const (GET|POST|PUT|PATCH|DELETE)\b/g, + ); + expect(handlers?.length ?? 0).toBeGreaterThan(0); + const auths = source.match(/await requireAuth\(\s*\)/g) ?? []; + const admits = source.match(/await admitDocumentSourceCaller\(/g) ?? []; + expect(auths.length).toBe(handlers!.length); + expect(admits.length).toBe(handlers!.length); + }, + ); + + it("the cookie check refuses every transport but the cookie", () => { + const support = stripComments( + readFileSync( + join(process.cwd(), "src/lib/documents/sources/route-support.ts"), + "utf8", + ), + ); + expect(support).toContain('auth.authMethod !== "cookie"'); + expect(support).toContain('"documents.sources.browserOnly"'); + }); +}); diff --git a/src/lib/openapi/routes/document-sources.ts b/src/lib/openapi/routes/document-sources.ts new file mode 100644 index 000000000..e25c0b813 --- /dev/null +++ b/src/lib/openapi/routes/document-sources.ts @@ -0,0 +1,402 @@ +/** + * OpenAPI route table for the document picker (`/api/documents/sources/*`, + * #1038): a person's connection to Paperless-ngx or Papra, searching it, and + * importing picked documents into the vault. + * + * Every operation is cookie-only (declared per operation as `cookieAuth`): a + * Bearer, even a `["*"]` one, is refused 403 `documents.sources.browserOnly`, + * and so is a request acting on another person's record. The native client does + * not use these routes. Request bodies reuse the runtime Zod schemas. + */ +import type { ZodOpenApiObject } from "zod-openapi"; +import { z } from "zod/v4"; + +import { + DOCUMENT_PICKER_LINK_KINDS, + DOCUMENT_PICKER_MAX_PAGE, + DOCUMENT_PICKER_QUERY_MAX, + DOCUMENT_PICKER_SYSTEMS, +} from "@/lib/documents/sources/types"; +import { + documentSourceImportSchema, + documentSourceSaveSchema, +} from "@/lib/validations/document-sources"; + +import { dataEnvelope, errorEnvelope, stdResponses } from "./shared"; + +const cookieOnly = [{ cookieAuth: [] }]; + +const systemParameter = { + name: "system", + in: "path" as const, + required: true, + description: "`paperless` (Paperless-ngx) or `papra` (Papra).", + schema: { type: "string" as const, enum: ["paperless", "papra"] }, +}; + +const systemEnum = z.enum(DOCUMENT_PICKER_SYSTEMS); + +const connectionSchema = z + .object({ + system: systemEnum, + baseUrl: z.string(), + organizationId: z.string().nullable(), + hasToken: z.literal(true), + lastVerifiedAt: z.string().nullable(), + originAllowed: z.boolean(), + }) + .meta({ id: "DocumentSourceConnection" }); + +const statusSchema = z + .object({ + available: z.boolean(), + allowedOrigins: z.array(z.string()), + connections: z.array(connectionSchema), + }) + .meta({ id: "DocumentSourcesStatus" }); + +const tagSchema = z + .object({ id: z.string(), name: z.string() }) + .meta({ id: "DocumentSourceTag" }); + +const resultSchema = z + .object({ + sourceId: z.string(), + title: z.string(), + date: z.string().nullable(), + tags: z.array(z.string()), + sizeBytes: z.number().int().nullable(), + state: z.enum(["new", "imported", "deleted"]), + documentId: z.string().nullable(), + }) + .meta({ id: "DocumentSourceResult" }); + +const searchSchema = z + .object({ + results: z.array(resultSchema), + page: z.number().int(), + hasMore: z.boolean(), + }) + .meta({ id: "DocumentSourceSearch" }); + +const importResultSchema = z + .object({ + outcome: z.enum(["imported", "duplicate", "deleted"]), + documentId: z.string().nullable(), + linked: z.boolean(), + }) + .meta({ id: "DocumentSourceImport" }); + +const refusal = { + description: + "Refused before any request to the source: `documents.sources.browserOnly` for a Bearer caller, `sharing.not_permitted` while acting on another person's record (a delegate or guardian never uses the owner's archive), or `module.disabled` when the Documents module is off.", + content: { "application/json": { schema: errorEnvelope } }, +}; + +const unavailable = { + description: + "`documents.sources.unavailable`: the operator has not listed any origin in `DOCUMENT_SOURCE_ORIGINS`, so the picker is off. `documents.sources.notConnected`: no connection is saved for this system. `documents.sources.notFound`: the source has no such document, or (Papra) no such organization.", + content: { "application/json": { schema: errorEnvelope } }, +}; + +const upstream = { + description: + "The source could not answer usefully, with the reason in `meta.errorCode` and, when the source answered at all, its HTTP status in `meta.upstreamStatus` (its body is never passed on): `documents.sources.unreachable` (no connection, timeout, or a destination refused at dial time), `documents.sources.redirected` (the source answered with a redirect, which is never followed), `documents.sources.authRefused` (401/403 from the source), `documents.sources.versionTooOld` (Paperless-ngx older than 2.16 / API version 9), `documents.sources.badResponse` (not the documented JSON, or larger than 1 MiB).", + content: { "application/json": { schema: errorEnvelope } }, +}; + +const originRefusal = { + description: + "`documents.sources.originNotAllowed`: the base address's origin is not on the operator's list (checked on every request, so a connection saved earlier is cut off once its origin is removed), or another validation failure.", + content: { "application/json": { schema: errorEnvelope } }, +}; + +export const documentSourcePaths: NonNullable = { + "/api/documents/sources": { + get: { + tags: ["Documents"], + summary: "Document archives: availability and connections", + description: + "Whether the operator enabled the document picker (`available`, true when `DOCUMENT_SOURCE_ORIGINS` lists at least one origin), the listed origins, and the caller's own saved connections to Paperless-ngx or Papra. A connection never carries its token: `hasToken` is always true for a saved one, and no route returns the token in whole or in part. `originAllowed` is false once the operator removed the connection's origin from the list. No request leaves the server. Cookie session only; not delegable.", + security: cookieOnly, + responses: { + "200": { + description: "The picker's state for the caller.", + content: { + "application/json": { + schema: dataEnvelope( + statusSchema, + "DocumentSourcesStatusEnvelope", + ), + }, + }, + }, + "403": refusal, + ...stdResponses, + }, + }, + }, + "/api/documents/sources/{system}": { + put: { + tags: ["Documents"], + summary: "Save a connection to a document archive", + description: + "Saves or replaces the caller's connection to Paperless-ngx (`paperless`: base address and API token) or Papra (`papra`: base address, organization id, API key with `documents:read` and `tags:read`). The base address may carry a path; its origin must be listed in `DOCUMENT_SOURCE_ORIGINS`. The connection is tested live before anything is stored. Leaving out `token` keeps the stored one (the first save must carry it). The token is stored encrypted (AES-256-GCM) and never returned. Ten saves and tests a minute per person. Cookie session only; not delegable.", + security: cookieOnly, + parameters: [systemParameter], + requestBody: { + required: true, + content: { "application/json": { schema: documentSourceSaveSchema } }, + }, + responses: { + "200": { + description: "Saved and verified. The connection, without its token.", + content: { + "application/json": { + schema: dataEnvelope( + connectionSchema, + "DocumentSourceConnectionEnvelope", + ), + }, + }, + }, + "403": refusal, + "404": unavailable, + "422": { + description: + "`documents.sources.originNotAllowed`, `documents.sources.invalidAddress`, `documents.sources.organizationRequired` (Papra), `documents.sources.tokenRequired` (first save without a token), or body validation.", + content: { "application/json": { schema: errorEnvelope } }, + }, + "502": upstream, + "401": stdResponses["401"], + "429": stdResponses["429"], + }, + }, + delete: { + tags: ["Documents"], + summary: "Remove a connection to a document archive", + description: + "Deletes the caller's connection for this system, and its token with it. Works even after the operator removed the origin from the list. `disconnected` is false when there was nothing to delete. Cookie session only; not delegable.", + security: cookieOnly, + parameters: [systemParameter], + responses: { + "200": { + description: "Removed, or there was nothing to remove.", + content: { + "application/json": { + schema: dataEnvelope( + z + .object({ disconnected: z.boolean() }) + .meta({ id: "DocumentSourceDisconnect" }), + "DocumentSourceDisconnectEnvelope", + ), + }, + }, + }, + "403": refusal, + ...stdResponses, + }, + }, + }, + "/api/documents/sources/{system}/test": { + post: { + tags: ["Documents"], + summary: "Test a saved connection", + description: + "One request to the source with the stored token. Answers `ok` with the round-trip time and stamps `lastVerifiedAt`, or the reason in `meta.errorCode`. Shares the save's allowance of ten a minute. Cookie session only; not delegable.", + security: cookieOnly, + parameters: [systemParameter], + responses: { + "200": { + description: "The source answered.", + content: { + "application/json": { + schema: dataEnvelope( + z + .object({ + ok: z.literal(true), + latencyMs: z.number().int(), + lastVerifiedAt: z.string(), + }) + .meta({ id: "DocumentSourceTest" }), + "DocumentSourceTestEnvelope", + ), + }, + }, + }, + "403": refusal, + "404": unavailable, + "422": originRefusal, + "502": upstream, + "401": stdResponses["401"], + "429": stdResponses["429"], + }, + }, + }, + "/api/documents/sources/{system}/tags": { + get: { + tags: ["Documents"], + summary: "Tags of a connected archive", + description: + "The source's tags for the picker's filter, the first 250 by name, in the source's own ids. Shares the search allowance of sixty a minute. Cookie session only; not delegable.", + security: cookieOnly, + parameters: [systemParameter], + responses: { + "200": { + description: "The tags.", + content: { + "application/json": { + schema: dataEnvelope( + z + .object({ tags: z.array(tagSchema) }) + .meta({ id: "DocumentSourceTags" }), + "DocumentSourceTagsEnvelope", + ), + }, + }, + }, + "403": refusal, + "404": unavailable, + "422": originRefusal, + "502": upstream, + "401": stdResponses["401"], + "429": stdResponses["429"], + }, + }, + }, + "/api/documents/sources/{system}/search": { + get: { + tags: ["Documents"], + summary: "Search a connected archive by name, tag and date", + description: + "Searches the source 25 documents at a time. `q` matches the document's name (Paperless-ngx title, Papra name); `tag` narrows to one of the source's tags by id; `from` / `to` to a date range, inclusive (Papra's range is applied to each page after it arrives, so a page can hold fewer than 25). Each result carries what the vault already holds under that source key: `new`, `imported` with the vault document's `documentId`, or `deleted` (deleted in HealthLog; importing it again stores nothing). Nothing is stored. Sixty searches a minute, shared with the tag read. Cookie session only; not delegable.", + security: cookieOnly, + parameters: [ + systemParameter, + { + name: "q", + in: "query", + required: false, + schema: { type: "string", maxLength: DOCUMENT_PICKER_QUERY_MAX }, + }, + { + name: "tag", + in: "query", + required: false, + schema: { type: "string", maxLength: 64 }, + }, + { + name: "from", + in: "query", + required: false, + schema: { type: "string", format: "date" }, + }, + { + name: "to", + in: "query", + required: false, + schema: { type: "string", format: "date" }, + }, + { + name: "page", + in: "query", + required: false, + schema: { + type: "integer", + minimum: 1, + maximum: DOCUMENT_PICKER_MAX_PAGE, + default: 1, + }, + }, + ], + responses: { + "200": { + description: "One page of results.", + content: { + "application/json": { + schema: dataEnvelope( + searchSchema, + "DocumentSourceSearchEnvelope", + ), + }, + }, + }, + "403": refusal, + "404": unavailable, + "422": originRefusal, + "502": upstream, + "401": stdResponses["401"], + "429": stdResponses["429"], + }, + }, + }, + "/api/documents/sources/{system}/import": { + post: { + tags: ["Documents"], + summary: "Import one picked document", + description: + "Fetches one document from the source and stores it in the vault through the same path as an upload, with `sourceSystem` / `sourceId` provenance. A source key the vault already holds is answered without a download: `duplicate` for a live document (same id), `deleted` for one the person deleted in HealthLog, which is never stored again. Bytes already in the vault under another key are `duplicate` too, and the key is remembered for that document. `link` files the document against the record the picker was opened from (a condition, a visit or a vaccination, which must be the caller's own and live; checked before anything is fetched), including a duplicate. `kind` sets the document type (default OTHER). Automatic AI reading follows the person's own setting. Counts against the person's upload allowance (60 an hour) and storage quota. 201 for a new document, 200 otherwise. Cookie session only; not delegable.", + security: cookieOnly, + parameters: [systemParameter], + requestBody: { + required: true, + content: { + "application/json": { schema: documentSourceImportSchema }, + }, + }, + responses: { + "201": { + description: "Imported.", + content: { + "application/json": { + schema: dataEnvelope( + importResultSchema, + "DocumentSourceImportEnvelope", + ), + }, + }, + }, + "200": { + description: + "Already held (`duplicate`) or deleted in HealthLog (`deleted`).", + content: { + "application/json": { + schema: dataEnvelope( + importResultSchema, + "DocumentSourceImportHeldEnvelope", + ), + }, + }, + }, + "403": refusal, + "404": { + description: `${unavailable.description} \`documents.sources.linkTargetNotFound\`: the record in \`link\` is not the caller's own live ${DOCUMENT_PICKER_LINK_KINDS.join(" / ")}.`, + content: { "application/json": { schema: errorEnvelope } }, + }, + "409": { + description: + "`documents.inbound.sourceAliasLimit`: these bytes are already stored under too many source ids.", + content: { "application/json": { schema: errorEnvelope } }, + }, + "413": { + description: + "`documents.inbound.fileTooLarge` (with `maxFileBytes`) or `documents.inbound.quotaExceeded` (with `quotaBytes` and `usedBytes`).", + content: { "application/json": { schema: errorEnvelope } }, + }, + "415": { + description: + "`documents.inbound.fileType`: the file is not a type the vault accepts.", + content: { "application/json": { schema: errorEnvelope } }, + }, + "422": originRefusal, + "502": upstream, + "401": stdResponses["401"], + "429": { + description: + "`documents.inbound.rateLimited` (the person's hourly upload allowance, with the rate-limit headers) or `documents.inbound.uploadBusy` (too many uploads in progress; `Retry-After: 1`).", + content: { "application/json": { schema: errorEnvelope } }, + }, + }, + }, + }, +}; diff --git a/src/lib/openapi/routes/index.ts b/src/lib/openapi/routes/index.ts index c1a5a0a21..377f5fb6c 100644 --- a/src/lib/openapi/routes/index.ts +++ b/src/lib/openapi/routes/index.ts @@ -50,6 +50,7 @@ import { cyclePaths } from "./cycle"; import { dashboardWidgetPaths } from "./dashboard"; import { biomarkerPaths } from "./biomarkers"; import { inboundDocumentPaths } from "./documents"; +import { documentSourcePaths } from "./document-sources"; import { devicePaths } from "./devices"; import { exportPaths } from "./export"; import { familyHistoryPaths } from "./family-history"; @@ -170,6 +171,10 @@ export const openApiPaths: NonNullable = { // discover: a copy this instance can no longer decrypt (appended, spread // order is load-bearing). ...adminBackupPaths, + // v1.39.3 — the document picker: a person's connection to Paperless-ngx or + // Papra, searching it and importing picked documents; cookie-only + // (appended, spread order is load-bearing). + ...documentSourcePaths, // The environmental-context overview. Its own module because nothing else // owns the surface (appended, spread order is load-bearing). // From 92f2cc729f7c06e21ac825f1493b76bfe310165c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Sat, 26 Sep 2026 13:18:48 +0200 Subject: [PATCH 06/18] fix(documents): name the document-AI capability in each route that can queue it, and catalogue the picker's error codes The ingest path takes the capability from its caller, so the upload and the import route each read getAiCapability("documentAi") in their own source, where the AI route inventory classifies them. --- docs/api/openapi.yaml | 9 ++++- src/app/api/__tests__/ai-route-inventory.ts | 5 +++ src/app/api/documents/inbound/route.ts | 4 ++ .../sources/[system]/import/route.ts | 4 ++ src/lib/documents/ingest.ts | 11 +++++- src/lib/documents/sources/types.ts | 39 ++++++++++++------- src/lib/openapi/error-codes.ts | 15 +++++++ 7 files changed, 68 insertions(+), 19 deletions(-) diff --git a/docs/api/openapi.yaml b/docs/api/openapi.yaml index 0827581f2..91f77e5c8 100644 --- a/docs/api/openapi.yaml +++ b/docs/api/openapi.yaml @@ -31198,8 +31198,13 @@ components: documents.inbound.notIndexed, documents.inbound.pdfNeedsAnthropic, documents.inbound.providerUnsupported, documents.inbound.quotaExceeded, documents.inbound.rateLimited, documents.inbound.restoreDuplicate, documents.inbound.restoreGone, documents.inbound.sourceAliasLimit, - documents.inbound.uploadBusy, documents.inbound.uploadTimeout, documents.inbound.vaccinationNotFound; - encounter — encounter.invalid, encounter.link.invalid, encounter.practitioner-not-found, + documents.inbound.uploadBusy, documents.inbound.uploadTimeout, documents.inbound.vaccinationNotFound, + documents.sources.authRefused, documents.sources.badResponse, documents.sources.browserOnly, + documents.sources.invalidAddress, documents.sources.linkTargetNotFound, documents.sources.notConnected, + documents.sources.notFound, documents.sources.organizationRequired, documents.sources.originNotAllowed, + documents.sources.rateLimited, documents.sources.redirected, documents.sources.tokenRequired, + documents.sources.unavailable, documents.sources.unreachable, documents.sources.versionTooOld; encounter + — encounter.invalid, encounter.link.invalid, encounter.practitioner-not-found, encounter.reminder-conflict, encounter.reminder-not-found; environment — environment.invalid, environment.no_home, environment.range_too_large, environment.travel.not_found; export — export.selection.empty, export.selection.unknown_leaf; familyHistory — familyHistory.invalid; feedback — diff --git a/src/app/api/__tests__/ai-route-inventory.ts b/src/app/api/__tests__/ai-route-inventory.ts index 2e49101b2..7e90462ed 100644 --- a/src/app/api/__tests__/ai-route-inventory.ts +++ b/src/app/api/__tests__/ai-route-inventory.ts @@ -67,6 +67,11 @@ export const AI_ROUTES: Readonly> = { capabilities: ["documentAi"], why: "The upload is data and always accepted; only the AI work queued after it follows the capability.", }, + "src/app/api/documents/sources/[system]/import/route.ts": { + kind: "mixed", + capabilities: ["documentAi"], + why: "A picked document is stored whatever AI says; only the summary queued after a fresh insert follows the capability.", + }, "src/app/api/auth/me/documents-auto-ai-read/route.ts": { kind: "mixed", capabilities: ["documentAi"], diff --git a/src/app/api/documents/inbound/route.ts b/src/app/api/documents/inbound/route.ts index f2f08dd2d..90c4dde70 100644 --- a/src/app/api/documents/inbound/route.ts +++ b/src/app/api/documents/inbound/route.ts @@ -49,6 +49,7 @@ import { getClientIp, sanitiseZodIssues, } from "@/lib/api-response"; +import { getAiCapability } from "@/lib/ai/capabilities/gate"; import { prisma } from "@/lib/db"; import { hashQueryTokens } from "@/lib/documents/content-index"; import { @@ -480,6 +481,9 @@ async function processUpload( aiDeferred: ctx.aiDeferred, sourceKeyChecked: true, limits, + // Only the AI work queued after a fresh insert follows the capability; + // the upload itself is data and always accepted. + documentAi: () => getAiCapability("documentAi"), }); switch (result.kind) { diff --git a/src/app/api/documents/sources/[system]/import/route.ts b/src/app/api/documents/sources/[system]/import/route.ts index a2cad851e..606e3c947 100644 --- a/src/app/api/documents/sources/[system]/import/route.ts +++ b/src/app/api/documents/sources/[system]/import/route.ts @@ -40,6 +40,7 @@ import { safeJson, sanitiseZodIssues, } from "@/lib/api-response"; +import { getAiCapability } from "@/lib/ai/capabilities/gate"; import { prisma } from "@/lib/db"; import { ingestDocument, @@ -241,6 +242,9 @@ export const POST = apiHandler( aiDeferred: false, sourceKeyChecked: true, limits, + // The import is data; only the summary queued after a fresh insert + // follows the capability (and the person's auto-read setting). + documentAi: () => getAiCapability("documentAi"), }); switch (result.kind) { diff --git a/src/lib/documents/ingest.ts b/src/lib/documents/ingest.ts index 06614a5c9..e38df19d3 100644 --- a/src/lib/documents/ingest.ts +++ b/src/lib/documents/ingest.ts @@ -29,7 +29,7 @@ import type { Buffer } from "node:buffer"; import { createHash } from "node:crypto"; -import { getAiCapability } from "@/lib/ai/capabilities/gate"; +import type { AiCapabilityState } from "@/lib/ai/capabilities/types"; import { auditLog } from "@/lib/auth/audit"; import { prisma } from "@/lib/db"; import { @@ -102,6 +102,13 @@ export interface IngestInput { sourceKeyChecked: boolean; /** The per-file cap and the quota, resolved by the caller. */ limits: { maxFileBytes: number; quotaBytes: number }; + /** + * The `documentAi` capability, asked by the caller's route only once a new + * document was stored (a duplicate never asks). Passed in rather than asked + * here so each route that can queue AI work names the capability in its own + * source, where the AI route inventory reads it. + */ + documentAi: () => Promise; } export type IngestResult = @@ -358,7 +365,7 @@ export async function ingestDocument( // the detail sheet offers "Generate summary" rather than a pending state. const documentAi = input.aiDeferred ? { available: false as const, reason: "deferred" } - : await getAiCapability("documentAi"); + : await input.documentAi(); if (documentAi.available) { void enqueueDocumentSummary(userId, document.id); } else { diff --git a/src/lib/documents/sources/types.ts b/src/lib/documents/sources/types.ts index 187f670b3..1e6cdf774 100644 --- a/src/lib/documents/sources/types.ts +++ b/src/lib/documents/sources/types.ts @@ -105,23 +105,32 @@ export const DOCUMENT_PICKER_LINK_KINDS = [ export type DocumentPickerLinkKind = (typeof DOCUMENT_PICKER_LINK_KINDS)[number]; -/** Why a request to a source failed, as `meta.errorCode` suffixes. */ +/** + * Why a request to a source failed, as the `meta.errorCode` values the routes + * emit. The route helper keys its status and wording on the last segment. + */ export const DOCUMENT_SOURCE_ERROR_CODES = [ - "unavailable", - "notConnected", - "originNotAllowed", - "unreachable", - "redirected", - "authRefused", - "versionTooOld", - "badResponse", - "notFound", - "linkTargetNotFound", - "rateLimited", + "documents.sources.unavailable", + "documents.sources.notConnected", + "documents.sources.originNotAllowed", + "documents.sources.unreachable", + "documents.sources.redirected", + "documents.sources.authRefused", + "documents.sources.versionTooOld", + "documents.sources.badResponse", + "documents.sources.notFound", + "documents.sources.linkTargetNotFound", + "documents.sources.rateLimited", ] as const; -export type DocumentSourceErrorCode = - (typeof DOCUMENT_SOURCE_ERROR_CODES)[number]; +type SourceErrorSuffix = T extends `documents.sources.${infer S}` + ? S + : never; +export type DocumentSourceErrorCode = SourceErrorSuffix< + (typeof DOCUMENT_SOURCE_ERROR_CODES)[number] +>; -export function documentSourceErrorCode(code: DocumentSourceErrorCode): string { +export function documentSourceErrorCode( + code: DocumentSourceErrorCode, +): (typeof DOCUMENT_SOURCE_ERROR_CODES)[number] { return `documents.sources.${code}`; } diff --git a/src/lib/openapi/error-codes.ts b/src/lib/openapi/error-codes.ts index 92602141d..f8bbf0e7b 100644 --- a/src/lib/openapi/error-codes.ts +++ b/src/lib/openapi/error-codes.ts @@ -136,6 +136,21 @@ export const ERROR_CODE_CATALOGUE: Readonly> = "documents.inbound.uploadBusy", "documents.inbound.uploadTimeout", "documents.inbound.vaccinationNotFound", + "documents.sources.authRefused", + "documents.sources.badResponse", + "documents.sources.browserOnly", + "documents.sources.invalidAddress", + "documents.sources.linkTargetNotFound", + "documents.sources.notConnected", + "documents.sources.notFound", + "documents.sources.organizationRequired", + "documents.sources.originNotAllowed", + "documents.sources.rateLimited", + "documents.sources.redirected", + "documents.sources.tokenRequired", + "documents.sources.unavailable", + "documents.sources.unreachable", + "documents.sources.versionTooOld", ], encounter: [ "encounter.invalid", From d109d67889a8d601bb9d3827f2e54d7384287bfa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Sat, 26 Sep 2026 13:26:18 +0200 Subject: [PATCH 07/18] feat(documents): pick documents from Paperless-ngx or Papra in the vault and wherever documents are linked A settings card under Integrations connects either system and tests it before saving. An import button beside Upload in Documents, on a condition's documents card, and beside Add in the visit and vaccination document pickers opens a sheet that searches by name with tag and date filters, shows what HealthLog already holds, and imports the ticked documents one at a time with a result per row. The buttons appear only for a person in their own record with a connection; strings in all seven languages. --- messages/de.json | 85 +++ messages/en.json | 85 +++ messages/es.json | 85 +++ messages/fr.json | 85 +++ messages/it.json | 85 +++ messages/ko.json | 85 +++ messages/pl.json | 85 +++ src/components/documents/documents-view.tsx | 19 +- .../documents/episode-documents-card.tsx | 4 + .../__tests__/document-sources-ui.test.tsx | 230 ++++++ .../sources/document-source-picker.tsx | 654 ++++++++++++++++++ .../sources/import-from-source-button.tsx | 73 ++ src/components/documents/sources/merge-ids.ts | 6 + .../documents/sources/source-errors.ts | 81 +++ .../documents/sources/use-document-sources.ts | 51 ++ .../encounters/encounter-link-pickers.tsx | 13 + src/components/links/entity-link-picker.tsx | 40 +- .../settings/integrations-section.tsx | 6 + .../integrations/document-sources-card.tsx | 415 +++++++++++ .../vaccination-document-picker.tsx | 10 + src/lib/query-keys/documents.ts | 23 + 21 files changed, 2200 insertions(+), 20 deletions(-) create mode 100644 src/components/documents/sources/__tests__/document-sources-ui.test.tsx create mode 100644 src/components/documents/sources/document-source-picker.tsx create mode 100644 src/components/documents/sources/import-from-source-button.tsx create mode 100644 src/components/documents/sources/merge-ids.ts create mode 100644 src/components/documents/sources/source-errors.ts create mode 100644 src/components/documents/sources/use-document-sources.ts create mode 100644 src/components/settings/integrations/document-sources-card.tsx diff --git a/messages/de.json b/messages/de.json index 06801e37e..b0b80d39f 100644 --- a/messages/de.json +++ b/messages/de.json @@ -6698,6 +6698,32 @@ "showChecklist": "Checkliste wieder anzeigen", "checklistShown": "Die Checkliste ist wieder auf dem Dashboard.", "checklistNothingHidden": "Die Checkliste war nicht ausgeblendet; sie öffnet sich auf dem Dashboard." + }, + "documentSources": { + "title": "Dokumentarchive", + "description": "Paperless-ngx oder Papra aus HealthLog durchsuchen und ausgewählte Dokumente importieren.", + "explainer": "Ein Import-Knopf erscheint überall, wo du Dokumente verknüpfst: bei Arztbesuchen, Impfungen und Erkrankungen sowie unter Dokumente. HealthLog speichert von jedem importierten Dokument eine eigene verschlüsselte Kopie und fragt dein Archiv nur, wenn du suchst oder importierst.", + "systemLabel": "Archiv", + "statusConnected": "Verbunden", + "statusNotConnected": "Nicht verbunden", + "connectedSince": "Verbunden, zuletzt geprüft am {date}", + "originRemoved": "Diese Adresse steht nicht mehr auf der Liste der erlaubten Archive dieses Servers. Frag die Administration oder speichere eine andere Adresse.", + "baseUrl": "Adresse", + "allowedOrigins": "Auf diesem Server erlaubt: {origins}", + "organizationId": "Organisations-ID", + "organizationHelp": "Der Teil nach /organizations/ in der Adresszeile von Papra.", + "token": "API-Token", + "tokenSaved": "Gespeichert. Leer lassen, um es zu behalten.", + "tokenHelpPaperless": "In Paperless-ngx: Mein Profil, API-Token.", + "tokenHelpPapra": "In Papra: ein API-Schlüssel mit documents:read und tags:read.", + "save": "Speichern und testen", + "saving": "Wird getestet …", + "saved": "Verbunden. Das Archiv hat geantwortet.", + "test": "Testen", + "testOk": "Das Archiv hat in {latency} ms geantwortet.", + "disconnect": "Trennen", + "disconnectTitle": "{name} trennen?", + "disconnectDescription": "HealthLog vergisst Adresse und Token. Bereits importierte Dokumente bleiben in HealthLog." } }, "admin": { @@ -10147,6 +10173,65 @@ "body": "Beim Lesen mit KI geht dies an deinen KI-Dienst außerhalb dieses Servers. Darf HealthLog Dokumente, Laborbefunde und Medikamentenbeschreibungen zum Lesen senden?", "grant": "Lesen erlauben", "error": "Die Einwilligung konnte nicht gespeichert werden. Bitte versuch es erneut." + }, + "sourcePicker": { + "open": "Aus Archiv importieren", + "openNamed": "Aus {name} importieren", + "title": "Aus {name} importieren", + "description": "Nach Namen suchen, dann die Dokumente anhaken, die HealthLog übernehmen soll.", + "sourceLabel": "Archiv", + "searchLabel": "Dokumentname", + "searchPlaceholder": "Nach Namen suchen", + "tagLabel": "Tag", + "tagAny": "Alle Tags", + "fromLabel": "Von", + "toLabel": "Bis", + "clearFilters": "Tag und Datum zurücksetzen", + "rangeInvalid": "Das Enddatum liegt vor dem Startdatum.", + "empty": "{name} hat keine Dokumente zum Anzeigen.", + "noMatch": "Keine passenden Dokumente.", + "loadMore": "Mehr laden", + "stateImported": "In HealthLog", + "stateDeleted": "In HealthLog gelöscht", + "selectedCount": "{count} von höchstens {max} ausgewählt", + "importAction": "{count} importieren", + "importing": "Import läuft, ein Dokument nach dem anderen …", + "done": "Fertig", + "summary": "Importiert: {imported}. Schon vorhanden: {present}. Nicht importiert: {failed}.", + "stopped": "Abgebrochen: {reason}", + "result": { + "waiting": "Wartet", + "working": "Wird importiert …", + "imported": "Importiert", + "importedLinked": "Importiert und verknüpft", + "duplicate": "Schon in HealthLog", + "duplicateLinked": "Schon in HealthLog, jetzt verknüpft", + "deleted": "In HealthLog gelöscht, bleibt gelöscht" + }, + "errors": { + "unavailable": "Dokumentarchive sind auf diesem Server nicht freigeschaltet.", + "notConnected": "Dieses Archiv ist nicht verbunden. Verbinde es unter Einstellungen, Integrationen.", + "originNotAllowed": "Diese Adresse steht nicht auf der Liste der erlaubten Archive dieses Servers. Bitte die Administration, sie aufzunehmen.", + "unreachable": "Das Archiv war nicht erreichbar. Prüfe, ob es läuft und die Adresse stimmt.", + "redirected": "Das Archiv hat die Anfrage umgeleitet. Trage die Adresse ein, auf die es umleitet.", + "authRefused": "Das Archiv hat das Token abgelehnt. Prüfe das Token und seine Berechtigungen.", + "versionTooOld": "Das Archiv ist zu alt. Nötig ist Paperless-ngx 2.16 oder neuer.", + "badResponse": "Die Antwort des Archivs ließ sich nicht lesen. Prüfe, ob die Adresse auf das Archiv selbst zeigt.", + "notFound": "Das Archiv kennt dieses Dokument oder diese Organisation nicht.", + "linkTargetNotFound": "Der Eintrag, mit dem verknüpft werden sollte, existiert nicht mehr.", + "rateLimited": "Zu viele Anfragen. Warte eine Minute und versuche es erneut.", + "browserOnly": "Dokumentarchive lassen sich nur im Browser verwenden.", + "invalidAddress": "Gib die vollständige Adresse ein, beginnend mit https:// oder http://.", + "organizationRequired": "Gib die Organisations-ID aus Papra ein.", + "tokenRequired": "Gib das API-Token ein.", + "fileTooLarge": "Die Datei ist größer, als HealthLog annimmt.", + "quotaExceeded": "Dein Dokumentenspeicher ist voll.", + "fileType": "HealthLog nimmt diesen Dateityp nicht an.", + "uploadLimit": "Das stündliche Upload-Limit ist erreicht. Versuche es später erneut.", + "uploadBusy": "Andere Uploads laufen noch. Versuche es gleich noch einmal.", + "moduleDisabled": "Dokumente sind in den Einstellungen ausgeschaltet.", + "generic": "Etwas ist schiefgelaufen. Versuche es erneut." + } } }, "selfDescription": { diff --git a/messages/en.json b/messages/en.json index fa1fb5458..324370d50 100644 --- a/messages/en.json +++ b/messages/en.json @@ -6698,6 +6698,32 @@ "showChecklist": "Show the checklist again", "checklistShown": "The checklist is back on the dashboard.", "checklistNothingHidden": "The checklist was not hidden; it opens on the dashboard." + }, + "documentSources": { + "title": "Document archives", + "description": "Search Paperless-ngx or Papra from HealthLog and import the documents you pick.", + "explainer": "An import button appears wherever you link documents: on a visit, a vaccination and a condition, and in Documents. HealthLog keeps its own encrypted copy of each document you import, and asks your archive only when you search or import.", + "systemLabel": "Archive", + "statusConnected": "Connected", + "statusNotConnected": "Not connected", + "connectedSince": "Connected, last checked {date}", + "originRemoved": "This address is no longer on the server's list of allowed archives. Ask the administrator, or save another address.", + "baseUrl": "Address", + "allowedOrigins": "Allowed on this server: {origins}", + "organizationId": "Organization id", + "organizationHelp": "The part after /organizations/ in the Papra address bar.", + "token": "API token", + "tokenSaved": "Saved. Leave empty to keep it.", + "tokenHelpPaperless": "In Paperless-ngx: My Profile, API token.", + "tokenHelpPapra": "In Papra: an API key with documents:read and tags:read.", + "save": "Save and test", + "saving": "Testing…", + "saved": "Connected. The archive answered.", + "test": "Test", + "testOk": "The archive answered in {latency} ms.", + "disconnect": "Disconnect", + "disconnectTitle": "Disconnect {name}?", + "disconnectDescription": "HealthLog forgets the address and the token. Documents you already imported stay in HealthLog." } }, "admin": { @@ -10147,6 +10173,65 @@ "body": "Reading with AI sends this to your AI service outside this server. Allow HealthLog to send documents, lab reports and medication descriptions for reading?", "grant": "Allow reading", "error": "The consent could not be saved. Please try again." + }, + "sourcePicker": { + "open": "Import from archive", + "openNamed": "Import from {name}", + "title": "Import from {name}", + "description": "Search by name, then tick the documents HealthLog should copy.", + "sourceLabel": "Archive", + "searchLabel": "Document name", + "searchPlaceholder": "Search by name", + "tagLabel": "Tag", + "tagAny": "Any tag", + "fromLabel": "From", + "toLabel": "To", + "clearFilters": "Clear tag and dates", + "rangeInvalid": "The end date is before the start date.", + "empty": "{name} has no documents to show.", + "noMatch": "No documents match.", + "loadMore": "Load more", + "stateImported": "In HealthLog", + "stateDeleted": "Deleted in HealthLog", + "selectedCount": "{count} of up to {max} selected", + "importAction": "Import {count}", + "importing": "Importing, one document at a time…", + "done": "Done", + "summary": "Imported: {imported}. Already there: {present}. Not imported: {failed}.", + "stopped": "Stopped: {reason}", + "result": { + "waiting": "Waiting", + "working": "Importing…", + "imported": "Imported", + "importedLinked": "Imported and linked", + "duplicate": "Already in HealthLog", + "duplicateLinked": "Already in HealthLog, now linked", + "deleted": "Deleted in HealthLog, left alone" + }, + "errors": { + "unavailable": "Document archives are not enabled on this server.", + "notConnected": "This archive is not connected. Connect it under Settings, Integrations.", + "originNotAllowed": "This address is not on the server's list of allowed archives. Ask the administrator to add it.", + "unreachable": "The archive could not be reached. Check that it is running and that the address is right.", + "redirected": "The archive redirected the request. Enter the address it redirects to.", + "authRefused": "The archive refused the token. Check the token and its permissions.", + "versionTooOld": "The archive is too old. Paperless-ngx 2.16 or later is needed.", + "badResponse": "The archive's answer could not be read. Check that the address points at the archive itself.", + "notFound": "The archive has no such document, or no such organization.", + "linkTargetNotFound": "The record to link to no longer exists.", + "rateLimited": "Too many requests. Wait a minute and try again.", + "browserOnly": "Document archives can only be used in the browser.", + "invalidAddress": "Enter the full address, starting with https:// or http://.", + "organizationRequired": "Enter the Papra organization id.", + "tokenRequired": "Enter the API token.", + "fileTooLarge": "The file is larger than HealthLog accepts.", + "quotaExceeded": "Your document storage is full.", + "fileType": "HealthLog does not accept this file type.", + "uploadLimit": "The hourly upload limit is reached. Try again later.", + "uploadBusy": "Other uploads are still running. Try again in a moment.", + "moduleDisabled": "Documents are switched off in Settings.", + "generic": "Something went wrong. Try again." + } } }, "selfDescription": { diff --git a/messages/es.json b/messages/es.json index 46f903102..f367a4433 100644 --- a/messages/es.json +++ b/messages/es.json @@ -6698,6 +6698,32 @@ "showChecklist": "Mostrar la lista de nuevo", "checklistShown": "La lista vuelve a estar en el panel.", "checklistNothingHidden": "La lista no estaba oculta; se abre en el panel." + }, + "documentSources": { + "title": "Archivos de documentos", + "description": "Busca en Paperless-ngx o Papra desde HealthLog e importa los documentos que elijas.", + "explainer": "Aparece un botón de importar allí donde vinculas documentos: en visitas, vacunas y afecciones, y en Documentos. HealthLog guarda su propia copia cifrada de cada documento importado y solo consulta tu archivo cuando buscas o importas.", + "systemLabel": "Archivo", + "statusConnected": "Conectado", + "statusNotConnected": "No conectado", + "connectedSince": "Conectado, última comprobación el {date}", + "originRemoved": "Esta dirección ya no está en la lista de archivos permitidos del servidor. Consulta a la administración o guarda otra dirección.", + "baseUrl": "Dirección", + "allowedOrigins": "Permitido en este servidor: {origins}", + "organizationId": "Id de organización", + "organizationHelp": "La parte que sigue a /organizations/ en la barra de direcciones de Papra.", + "token": "Token de la API", + "tokenSaved": "Guardado. Déjalo vacío para conservarlo.", + "tokenHelpPaperless": "En Paperless-ngx: Mi perfil, token de la API.", + "tokenHelpPapra": "En Papra: una clave de API con documents:read y tags:read.", + "save": "Guardar y probar", + "saving": "Probando…", + "saved": "Conectado. El archivo respondió.", + "test": "Probar", + "testOk": "El archivo respondió en {latency} ms.", + "disconnect": "Desconectar", + "disconnectTitle": "¿Desconectar {name}?", + "disconnectDescription": "HealthLog olvida la dirección y el token. Los documentos ya importados se quedan en HealthLog." } }, "admin": { @@ -10147,6 +10173,65 @@ "body": "Leer con IA envía esto a tu servicio de IA fuera de este servidor. ¿Permitir que HealthLog envíe documentos, informes de laboratorio y descripciones de medicamentos para leerlos?", "grant": "Permitir lectura", "error": "No se pudo guardar el consentimiento. Inténtalo de nuevo." + }, + "sourcePicker": { + "open": "Importar desde archivo", + "openNamed": "Importar desde {name}", + "title": "Importar desde {name}", + "description": "Busca por nombre y marca los documentos que HealthLog debe copiar.", + "sourceLabel": "Archivo", + "searchLabel": "Nombre del documento", + "searchPlaceholder": "Buscar por nombre", + "tagLabel": "Etiqueta", + "tagAny": "Cualquier etiqueta", + "fromLabel": "Desde", + "toLabel": "Hasta", + "clearFilters": "Quitar etiqueta y fechas", + "rangeInvalid": "La fecha final es anterior a la inicial.", + "empty": "{name} no tiene documentos que mostrar.", + "noMatch": "Ningún documento coincide.", + "loadMore": "Cargar más", + "stateImported": "En HealthLog", + "stateDeleted": "Eliminado en HealthLog", + "selectedCount": "{count} de un máximo de {max} seleccionados", + "importAction": "Importar {count}", + "importing": "Importando, un documento tras otro…", + "done": "Hecho", + "summary": "Importados: {imported}. Ya estaban: {present}. No importados: {failed}.", + "stopped": "Detenido: {reason}", + "result": { + "waiting": "En espera", + "working": "Importando…", + "imported": "Importado", + "importedLinked": "Importado y vinculado", + "duplicate": "Ya está en HealthLog", + "duplicateLinked": "Ya está en HealthLog, ahora vinculado", + "deleted": "Eliminado en HealthLog, se deja así" + }, + "errors": { + "unavailable": "Los archivos de documentos no están habilitados en este servidor.", + "notConnected": "Este archivo no está conectado. Conéctalo en Ajustes, Integraciones.", + "originNotAllowed": "Esta dirección no está en la lista de archivos permitidos del servidor. Pide a la administración que la añada.", + "unreachable": "No se pudo acceder al archivo. Comprueba que está en marcha y que la dirección es correcta.", + "redirected": "El archivo redirigió la solicitud. Introduce la dirección a la que redirige.", + "authRefused": "El archivo rechazó el token. Comprueba el token y sus permisos.", + "versionTooOld": "El archivo es demasiado antiguo. Se necesita Paperless-ngx 2.16 o posterior.", + "badResponse": "No se pudo leer la respuesta del archivo. Comprueba que la dirección apunta al propio archivo.", + "notFound": "El archivo no tiene ese documento o esa organización.", + "linkTargetNotFound": "El registro que se iba a vincular ya no existe.", + "rateLimited": "Demasiadas solicitudes. Espera un minuto y vuelve a intentarlo.", + "browserOnly": "Los archivos de documentos solo se pueden usar en el navegador.", + "invalidAddress": "Introduce la dirección completa, empezando por https:// o http://.", + "organizationRequired": "Introduce el id de organización de Papra.", + "tokenRequired": "Introduce el token de la API.", + "fileTooLarge": "El archivo supera el tamaño que acepta HealthLog.", + "quotaExceeded": "Tu almacenamiento de documentos está lleno.", + "fileType": "HealthLog no acepta este tipo de archivo.", + "uploadLimit": "Se alcanzó el límite de subidas por hora. Inténtalo más tarde.", + "uploadBusy": "Aún hay otras subidas en curso. Inténtalo en un momento.", + "moduleDisabled": "Los documentos están desactivados en Ajustes.", + "generic": "Algo salió mal. Inténtalo de nuevo." + } } }, "selfDescription": { diff --git a/messages/fr.json b/messages/fr.json index 722b77a24..1037939c5 100644 --- a/messages/fr.json +++ b/messages/fr.json @@ -6698,6 +6698,32 @@ "showChecklist": "Réafficher la liste de départ", "checklistShown": "La liste de départ est de retour sur le tableau de bord.", "checklistNothingHidden": "La liste n'était pas masquée ; elle s'ouvre sur le tableau de bord." + }, + "documentSources": { + "title": "Archives de documents", + "description": "Cherchez dans Paperless-ngx ou Papra depuis HealthLog et importez les documents choisis.", + "explainer": "Un bouton d'import apparaît partout où vous liez des documents : consultations, vaccinations, affections et Documents. HealthLog garde sa propre copie chiffrée de chaque document importé et n'interroge votre archive que lorsque vous cherchez ou importez.", + "systemLabel": "Archive", + "statusConnected": "Connecté", + "statusNotConnected": "Non connecté", + "connectedSince": "Connecté, dernière vérification le {date}", + "originRemoved": "Cette adresse ne figure plus dans la liste des archives autorisées du serveur. Voyez avec l'administration ou enregistrez une autre adresse.", + "baseUrl": "Adresse", + "allowedOrigins": "Autorisé sur ce serveur : {origins}", + "organizationId": "Identifiant d'organisation", + "organizationHelp": "La partie après /organizations/ dans la barre d'adresse de Papra.", + "token": "Jeton d'API", + "tokenSaved": "Enregistré. Laissez vide pour le conserver.", + "tokenHelpPaperless": "Dans Paperless-ngx : Mon profil, jeton d'API.", + "tokenHelpPapra": "Dans Papra : une clé d'API avec documents:read et tags:read.", + "save": "Enregistrer et tester", + "saving": "Test en cours…", + "saved": "Connecté. L'archive a répondu.", + "test": "Tester", + "testOk": "L'archive a répondu en {latency} ms.", + "disconnect": "Déconnecter", + "disconnectTitle": "Déconnecter {name} ?", + "disconnectDescription": "HealthLog oublie l'adresse et le jeton. Les documents déjà importés restent dans HealthLog." } }, "admin": { @@ -10147,6 +10173,65 @@ "body": "La lecture par IA envoie ceci à votre service d'IA hors de ce serveur. Autoriser HealthLog à envoyer des documents, des résultats d'analyses et des descriptions de médicaments pour lecture ?", "grant": "Autoriser la lecture", "error": "Le consentement n'a pas pu être enregistré. Veuillez réessayer." + }, + "sourcePicker": { + "open": "Importer depuis une archive", + "openNamed": "Importer depuis {name}", + "title": "Importer depuis {name}", + "description": "Cherchez par nom, puis cochez les documents que HealthLog doit copier.", + "sourceLabel": "Archive", + "searchLabel": "Nom du document", + "searchPlaceholder": "Chercher par nom", + "tagLabel": "Étiquette", + "tagAny": "Toutes les étiquettes", + "fromLabel": "Du", + "toLabel": "Au", + "clearFilters": "Effacer l'étiquette et les dates", + "rangeInvalid": "La date de fin précède la date de début.", + "empty": "{name} n'a aucun document à afficher.", + "noMatch": "Aucun document ne correspond.", + "loadMore": "Charger plus", + "stateImported": "Dans HealthLog", + "stateDeleted": "Supprimé dans HealthLog", + "selectedCount": "{count} sélectionnés sur {max} au plus", + "importAction": "Importer {count}", + "importing": "Import en cours, un document après l'autre…", + "done": "Terminé", + "summary": "Importés : {imported}. Déjà présents : {present}. Non importés : {failed}.", + "stopped": "Arrêté : {reason}", + "result": { + "waiting": "En attente", + "working": "Import en cours…", + "imported": "Importé", + "importedLinked": "Importé et lié", + "duplicate": "Déjà dans HealthLog", + "duplicateLinked": "Déjà dans HealthLog, désormais lié", + "deleted": "Supprimé dans HealthLog, laissé tel quel" + }, + "errors": { + "unavailable": "Les archives de documents ne sont pas activées sur ce serveur.", + "notConnected": "Cette archive n'est pas connectée. Connectez-la dans Réglages, Intégrations.", + "originNotAllowed": "Cette adresse ne figure pas dans la liste des archives autorisées du serveur. Demandez à l'administration de l'ajouter.", + "unreachable": "L'archive est injoignable. Vérifiez qu'elle fonctionne et que l'adresse est juste.", + "redirected": "L'archive a redirigé la requête. Saisissez l'adresse vers laquelle elle redirige.", + "authRefused": "L'archive a refusé le jeton. Vérifiez le jeton et ses droits.", + "versionTooOld": "L'archive est trop ancienne. Paperless-ngx 2.16 ou plus récent est nécessaire.", + "badResponse": "La réponse de l'archive est illisible. Vérifiez que l'adresse pointe vers l'archive elle-même.", + "notFound": "L'archive ne connaît pas ce document ou cette organisation.", + "linkTargetNotFound": "L'élément auquel lier le document n'existe plus.", + "rateLimited": "Trop de requêtes. Patientez une minute et réessayez.", + "browserOnly": "Les archives de documents ne s'utilisent que dans le navigateur.", + "invalidAddress": "Saisissez l'adresse complète, commençant par https:// ou http://.", + "organizationRequired": "Saisissez l'identifiant d'organisation Papra.", + "tokenRequired": "Saisissez le jeton d'API.", + "fileTooLarge": "Le fichier dépasse la taille acceptée par HealthLog.", + "quotaExceeded": "Votre espace de documents est plein.", + "fileType": "HealthLog n'accepte pas ce type de fichier.", + "uploadLimit": "La limite horaire d'envois est atteinte. Réessayez plus tard.", + "uploadBusy": "D'autres envois sont en cours. Réessayez dans un instant.", + "moduleDisabled": "Les documents sont désactivés dans les réglages.", + "generic": "Un problème est survenu. Réessayez." + } } }, "selfDescription": { diff --git a/messages/it.json b/messages/it.json index ae8310f2d..1d4c709f7 100644 --- a/messages/it.json +++ b/messages/it.json @@ -6698,6 +6698,32 @@ "showChecklist": "Mostra di nuovo la lista", "checklistShown": "La lista è di nuovo nella dashboard.", "checklistNothingHidden": "La lista non era nascosta; si apre nella dashboard." + }, + "documentSources": { + "title": "Archivi di documenti", + "description": "Cerca in Paperless-ngx o Papra da HealthLog e importa i documenti che scegli.", + "explainer": "Un pulsante di importazione compare ovunque colleghi documenti: nelle visite, nelle vaccinazioni, nelle condizioni e in Documenti. HealthLog conserva una propria copia cifrata di ogni documento importato e interroga il tuo archivio solo quando cerchi o importi.", + "systemLabel": "Archivio", + "statusConnected": "Collegato", + "statusNotConnected": "Non collegato", + "connectedSince": "Collegato, ultimo controllo il {date}", + "originRemoved": "Questo indirizzo non è più nell'elenco degli archivi consentiti dal server. Chiedi all'amministrazione o salva un altro indirizzo.", + "baseUrl": "Indirizzo", + "allowedOrigins": "Consentito su questo server: {origins}", + "organizationId": "Id organizzazione", + "organizationHelp": "La parte dopo /organizations/ nella barra degli indirizzi di Papra.", + "token": "Token API", + "tokenSaved": "Salvato. Lascia vuoto per mantenerlo.", + "tokenHelpPaperless": "In Paperless-ngx: Il mio profilo, token API.", + "tokenHelpPapra": "In Papra: una chiave API con documents:read e tags:read.", + "save": "Salva e prova", + "saving": "Prova in corso…", + "saved": "Collegato. L'archivio ha risposto.", + "test": "Prova", + "testOk": "L'archivio ha risposto in {latency} ms.", + "disconnect": "Scollega", + "disconnectTitle": "Scollegare {name}?", + "disconnectDescription": "HealthLog dimentica indirizzo e token. I documenti già importati restano in HealthLog." } }, "admin": { @@ -10147,6 +10173,65 @@ "body": "La lettura con IA invia questo contenuto al tuo servizio di IA fuori da questo server. Consentire a HealthLog di inviare documenti, referti di laboratorio e descrizioni di farmaci per la lettura?", "grant": "Consenti lettura", "error": "Non è stato possibile salvare il consenso. Riprova." + }, + "sourcePicker": { + "open": "Importa da archivio", + "openNamed": "Importa da {name}", + "title": "Importa da {name}", + "description": "Cerca per nome, poi spunta i documenti che HealthLog deve copiare.", + "sourceLabel": "Archivio", + "searchLabel": "Nome del documento", + "searchPlaceholder": "Cerca per nome", + "tagLabel": "Etichetta", + "tagAny": "Qualsiasi etichetta", + "fromLabel": "Dal", + "toLabel": "Al", + "clearFilters": "Azzera etichetta e date", + "rangeInvalid": "La data finale è precedente a quella iniziale.", + "empty": "{name} non ha documenti da mostrare.", + "noMatch": "Nessun documento corrisponde.", + "loadMore": "Carica altri", + "stateImported": "Già in HealthLog", + "stateDeleted": "Eliminato in HealthLog", + "selectedCount": "{count} selezionati su un massimo di {max}", + "importAction": "Importa {count}", + "importing": "Importazione in corso, un documento alla volta…", + "done": "Fatto", + "summary": "Importati: {imported}. Già presenti: {present}. Non importati: {failed}.", + "stopped": "Interrotto: {reason}", + "result": { + "waiting": "In attesa", + "working": "Importazione…", + "imported": "Importato", + "importedLinked": "Importato e collegato", + "duplicate": "Già in HealthLog", + "duplicateLinked": "Già in HealthLog, ora collegato", + "deleted": "Eliminato in HealthLog, lasciato così" + }, + "errors": { + "unavailable": "Gli archivi di documenti non sono abilitati su questo server.", + "notConnected": "Questo archivio non è collegato. Collegalo in Impostazioni, Integrazioni.", + "originNotAllowed": "Questo indirizzo non è nell'elenco degli archivi consentiti dal server. Chiedi all'amministrazione di aggiungerlo.", + "unreachable": "L'archivio non è raggiungibile. Verifica che sia in funzione e che l'indirizzo sia corretto.", + "redirected": "L'archivio ha reindirizzato la richiesta. Inserisci l'indirizzo verso cui reindirizza.", + "authRefused": "L'archivio ha rifiutato il token. Controlla il token e i suoi permessi.", + "versionTooOld": "L'archivio è troppo vecchio. Serve Paperless-ngx 2.16 o successivo.", + "badResponse": "Non è stato possibile leggere la risposta dell'archivio. Verifica che l'indirizzo punti all'archivio stesso.", + "notFound": "L'archivio non ha questo documento o questa organizzazione.", + "linkTargetNotFound": "L'elemento da collegare non esiste più.", + "rateLimited": "Troppe richieste. Attendi un minuto e riprova.", + "browserOnly": "Gli archivi di documenti si usano solo nel browser.", + "invalidAddress": "Inserisci l'indirizzo completo, che inizia con https:// o http://.", + "organizationRequired": "Inserisci l'id dell'organizzazione Papra.", + "tokenRequired": "Inserisci il token API.", + "fileTooLarge": "Il file è più grande di quanto HealthLog accetti.", + "quotaExceeded": "Lo spazio per i documenti è pieno.", + "fileType": "HealthLog non accetta questo tipo di file.", + "uploadLimit": "È stato raggiunto il limite orario di caricamenti. Riprova più tardi.", + "uploadBusy": "Altri caricamenti sono ancora in corso. Riprova tra un momento.", + "moduleDisabled": "I documenti sono disattivati nelle impostazioni.", + "generic": "Qualcosa è andato storto. Riprova." + } } }, "selfDescription": { diff --git a/messages/ko.json b/messages/ko.json index 674b6c9a3..42c93bcf7 100644 --- a/messages/ko.json +++ b/messages/ko.json @@ -6698,6 +6698,32 @@ "showChecklist": "체크리스트 다시 표시", "checklistShown": "체크리스트가 대시보드에 다시 표시됩니다.", "checklistNothingHidden": "체크리스트는 숨겨져 있지 않았습니다. 대시보드에서 열립니다." + }, + "documentSources": { + "title": "문서 보관함", + "description": "HealthLog에서 Paperless-ngx나 Papra를 검색하고 선택한 문서를 가져옵니다.", + "explainer": "문서를 연결하는 곳마다 가져오기 버튼이 나타납니다: 진료, 예방접종, 질환, 그리고 문서 페이지. HealthLog는 가져온 문서마다 자체 암호화 사본을 보관하며, 검색하거나 가져올 때만 보관함에 요청합니다.", + "systemLabel": "보관함", + "statusConnected": "연결됨", + "statusNotConnected": "연결 안 됨", + "connectedSince": "연결됨, 마지막 확인 {date}", + "originRemoved": "이 주소는 더 이상 서버의 허용된 보관함 목록에 없습니다. 관리자에게 문의하거나 다른 주소를 저장하세요.", + "baseUrl": "주소", + "allowedOrigins": "이 서버에서 허용됨: {origins}", + "organizationId": "조직 ID", + "organizationHelp": "Papra 주소 표시줄에서 /organizations/ 다음 부분입니다.", + "token": "API 토큰", + "tokenSaved": "저장됨. 유지하려면 비워 두세요.", + "tokenHelpPaperless": "Paperless-ngx: 내 프로필의 API 토큰.", + "tokenHelpPapra": "Papra: documents:read와 tags:read 권한이 있는 API 키.", + "save": "저장하고 테스트", + "saving": "테스트 중…", + "saved": "연결되었습니다. 보관함이 응답했습니다.", + "test": "테스트", + "testOk": "보관함이 {latency}ms 만에 응답했습니다.", + "disconnect": "연결 해제", + "disconnectTitle": "{name} 연결을 해제할까요?", + "disconnectDescription": "HealthLog가 주소와 토큰을 잊습니다. 이미 가져온 문서는 HealthLog에 남습니다." } }, "admin": { @@ -10147,6 +10173,65 @@ "body": "AI로 읽으면 이 내용이 이 서버 밖의 AI 서비스로 전송됩니다. HealthLog가 문서, 검사 결과지, 약 설명을 읽기용으로 보내도록 허용하시겠습니까?", "grant": "읽기 허용", "error": "동의를 저장하지 못했습니다. 다시 시도해 주세요." + }, + "sourcePicker": { + "open": "보관함에서 가져오기", + "openNamed": "{name}에서 가져오기", + "title": "{name}에서 가져오기", + "description": "이름으로 검색한 뒤 HealthLog로 복사할 문서를 선택하세요.", + "sourceLabel": "보관함", + "searchLabel": "문서 이름", + "searchPlaceholder": "이름으로 검색", + "tagLabel": "태그", + "tagAny": "모든 태그", + "fromLabel": "시작일", + "toLabel": "종료일", + "clearFilters": "태그와 날짜 지우기", + "rangeInvalid": "종료일이 시작일보다 앞섭니다.", + "empty": "{name}에 표시할 문서가 없습니다.", + "noMatch": "일치하는 문서가 없습니다.", + "loadMore": "더 불러오기", + "stateImported": "HealthLog에 있음", + "stateDeleted": "HealthLog에서 삭제됨", + "selectedCount": "최대 {max}개 중 {count}개 선택", + "importAction": "{count}개 가져오기", + "importing": "문서를 하나씩 가져오는 중…", + "done": "완료", + "summary": "가져옴: {imported}. 이미 있음: {present}. 가져오지 못함: {failed}.", + "stopped": "중단됨: {reason}", + "result": { + "waiting": "대기 중", + "working": "가져오는 중…", + "imported": "가져옴", + "importedLinked": "가져오고 연결함", + "duplicate": "이미 HealthLog에 있음", + "duplicateLinked": "이미 HealthLog에 있음, 이제 연결됨", + "deleted": "HealthLog에서 삭제됨, 그대로 둠" + }, + "errors": { + "unavailable": "이 서버에서는 문서 보관함이 활성화되어 있지 않습니다.", + "notConnected": "이 보관함은 연결되어 있지 않습니다. 설정의 통합에서 연결하세요.", + "originNotAllowed": "이 주소는 서버의 허용된 보관함 목록에 없습니다. 관리자에게 추가를 요청하세요.", + "unreachable": "보관함에 연결할 수 없습니다. 실행 중인지, 주소가 맞는지 확인하세요.", + "redirected": "보관함이 요청을 리디렉션했습니다. 리디렉션되는 주소를 입력하세요.", + "authRefused": "보관함이 토큰을 거부했습니다. 토큰과 권한을 확인하세요.", + "versionTooOld": "보관함 버전이 너무 오래되었습니다. Paperless-ngx 2.16 이상이 필요합니다.", + "badResponse": "보관함의 응답을 읽을 수 없습니다. 주소가 보관함 자체를 가리키는지 확인하세요.", + "notFound": "보관함에 해당 문서나 조직이 없습니다.", + "linkTargetNotFound": "연결할 기록이 더 이상 존재하지 않습니다.", + "rateLimited": "요청이 너무 많습니다. 1분 후에 다시 시도하세요.", + "browserOnly": "문서 보관함은 브라우저에서만 사용할 수 있습니다.", + "invalidAddress": "https:// 또는 http://로 시작하는 전체 주소를 입력하세요.", + "organizationRequired": "Papra 조직 ID를 입력하세요.", + "tokenRequired": "API 토큰을 입력하세요.", + "fileTooLarge": "파일이 HealthLog가 허용하는 크기보다 큽니다.", + "quotaExceeded": "문서 저장 공간이 가득 찼습니다.", + "fileType": "HealthLog는 이 파일 형식을 받지 않습니다.", + "uploadLimit": "시간당 업로드 한도에 도달했습니다. 나중에 다시 시도하세요.", + "uploadBusy": "다른 업로드가 아직 진행 중입니다. 잠시 후 다시 시도하세요.", + "moduleDisabled": "설정에서 문서 기능이 꺼져 있습니다.", + "generic": "문제가 발생했습니다. 다시 시도하세요." + } } }, "selfDescription": { diff --git a/messages/pl.json b/messages/pl.json index a2c23d8b3..680921471 100644 --- a/messages/pl.json +++ b/messages/pl.json @@ -6698,6 +6698,32 @@ "showChecklist": "Pokaż listę ponownie", "checklistShown": "Lista wróciła na panel.", "checklistNothingHidden": "Lista nie była ukryta; otwiera się na panelu." + }, + "documentSources": { + "title": "Archiwa dokumentów", + "description": "Przeszukuj Paperless-ngx lub Papra z poziomu HealthLog i importuj wybrane dokumenty.", + "explainer": "Przycisk importu pojawia się wszędzie tam, gdzie wiążesz dokumenty: przy wizytach, szczepieniach i schorzeniach oraz w Dokumentach. HealthLog przechowuje własną zaszyfrowaną kopię każdego zaimportowanego dokumentu i pyta archiwum tylko wtedy, gdy szukasz lub importujesz.", + "systemLabel": "Archiwum", + "statusConnected": "Połączono", + "statusNotConnected": "Nie połączono", + "connectedSince": "Połączono, ostatnio sprawdzono {date}", + "originRemoved": "Tego adresu nie ma już na liście dozwolonych archiwów serwera. Zapytaj administrację albo zapisz inny adres.", + "baseUrl": "Adres", + "allowedOrigins": "Dozwolone na tym serwerze: {origins}", + "organizationId": "Identyfikator organizacji", + "organizationHelp": "Część po /organizations/ w pasku adresu Papra.", + "token": "Token API", + "tokenSaved": "Zapisano. Zostaw puste, aby go zachować.", + "tokenHelpPaperless": "W Paperless-ngx: Mój profil, token API.", + "tokenHelpPapra": "W Papra: klucz API z uprawnieniami documents:read i tags:read.", + "save": "Zapisz i sprawdź", + "saving": "Sprawdzanie…", + "saved": "Połączono. Archiwum odpowiedziało.", + "test": "Sprawdź", + "testOk": "Archiwum odpowiedziało w {latency} ms.", + "disconnect": "Rozłącz", + "disconnectTitle": "Rozłączyć {name}?", + "disconnectDescription": "HealthLog zapomina adres i token. Zaimportowane już dokumenty zostają w HealthLog." } }, "admin": { @@ -10147,6 +10173,65 @@ "body": "Odczyt przez AI wysyła to do Twojej usługi AI poza tym serwerem. Czy HealthLog może wysyłać dokumenty, wyniki badań i opisy leków do odczytu?", "grant": "Zezwól na odczyt", "error": "Nie udało się zapisać zgody. Spróbuj ponownie." + }, + "sourcePicker": { + "open": "Importuj z archiwum", + "openNamed": "Importuj z {name}", + "title": "Import z {name}", + "description": "Szukaj po nazwie, a potem zaznacz dokumenty, które HealthLog ma skopiować.", + "sourceLabel": "Archiwum", + "searchLabel": "Nazwa dokumentu", + "searchPlaceholder": "Szukaj po nazwie", + "tagLabel": "Tag", + "tagAny": "Dowolny tag", + "fromLabel": "Od", + "toLabel": "Do", + "clearFilters": "Wyczyść tag i daty", + "rangeInvalid": "Data końcowa jest wcześniejsza niż początkowa.", + "empty": "{name} nie ma dokumentów do pokazania.", + "noMatch": "Brak pasujących dokumentów.", + "loadMore": "Wczytaj więcej", + "stateImported": "W HealthLog", + "stateDeleted": "Usunięty w HealthLog", + "selectedCount": "Wybrano {count} z maksymalnie {max}", + "importAction": "Importuj ({count})", + "importing": "Importowanie, dokument po dokumencie…", + "done": "Gotowe", + "summary": "Zaimportowane: {imported}. Już były: {present}. Niezaimportowane: {failed}.", + "stopped": "Zatrzymano: {reason}", + "result": { + "waiting": "Czeka", + "working": "Importowanie…", + "imported": "Zaimportowano", + "importedLinked": "Zaimportowano i powiązano", + "duplicate": "Już jest w HealthLog", + "duplicateLinked": "Już jest w HealthLog, teraz powiązany", + "deleted": "Usunięty w HealthLog, pozostaje usunięty" + }, + "errors": { + "unavailable": "Archiwa dokumentów nie są włączone na tym serwerze.", + "notConnected": "To archiwum nie jest połączone. Połącz je w Ustawieniach, Integracje.", + "originNotAllowed": "Tego adresu nie ma na liście dozwolonych archiwów serwera. Poproś administrację o jego dodanie.", + "unreachable": "Nie udało się połączyć z archiwum. Sprawdź, czy działa i czy adres jest poprawny.", + "redirected": "Archiwum przekierowało żądanie. Wpisz adres, na który przekierowuje.", + "authRefused": "Archiwum odrzuciło token. Sprawdź token i jego uprawnienia.", + "versionTooOld": "Archiwum jest za stare. Potrzebny jest Paperless-ngx 2.16 lub nowszy.", + "badResponse": "Nie udało się odczytać odpowiedzi archiwum. Sprawdź, czy adres wskazuje na samo archiwum.", + "notFound": "Archiwum nie ma takiego dokumentu ani takiej organizacji.", + "linkTargetNotFound": "Wpis, z którym miało nastąpić powiązanie, już nie istnieje.", + "rateLimited": "Za dużo żądań. Odczekaj minutę i spróbuj ponownie.", + "browserOnly": "Z archiwów dokumentów można korzystać tylko w przeglądarce.", + "invalidAddress": "Wpisz pełny adres, zaczynający się od https:// lub http://.", + "organizationRequired": "Wpisz identyfikator organizacji Papra.", + "tokenRequired": "Wpisz token API.", + "fileTooLarge": "Plik jest większy, niż przyjmuje HealthLog.", + "quotaExceeded": "Miejsce na dokumenty jest pełne.", + "fileType": "HealthLog nie przyjmuje tego typu pliku.", + "uploadLimit": "Osiągnięto godzinny limit przesyłania. Spróbuj później.", + "uploadBusy": "Trwają jeszcze inne przesyłania. Spróbuj za chwilę.", + "moduleDisabled": "Dokumenty są wyłączone w ustawieniach.", + "generic": "Coś poszło nie tak. Spróbuj ponownie." + } } }, "selfDescription": { diff --git a/src/components/documents/documents-view.tsx b/src/components/documents/documents-view.tsx index 67e5ac4aa..c68bb8f5f 100644 --- a/src/components/documents/documents-view.tsx +++ b/src/components/documents/documents-view.tsx @@ -67,6 +67,7 @@ import { DocumentShareSheet } from "./document-share-sheet"; import { DocumentFilterBar, type ConditionChip } from "./document-filter-bar"; import { DocumentTimeline } from "./document-timeline"; import { UploadZone } from "./upload-zone"; +import { ImportFromSourceButton } from "./sources/import-from-source-button"; import { useReindexAll } from "./use-content-index"; import { useDocumentUpload } from "./use-document-upload"; import { usePageFileDrop } from "./use-page-file-drop"; @@ -876,13 +877,17 @@ export function DocumentsView() { description={t("documents.subtitle")} actions={ canManageDocuments ? ( - +
+ {/* Renders only with a connected Paperless-ngx or Papra. */} + + +
) : null } /> diff --git a/src/components/documents/episode-documents-card.tsx b/src/components/documents/episode-documents-card.tsx index fc42f410b..2c9bc580f 100644 --- a/src/components/documents/episode-documents-card.tsx +++ b/src/components/documents/episode-documents-card.tsx @@ -32,6 +32,7 @@ import { useFormatters, useTranslations } from "@/lib/i18n/context"; import { queryKeys } from "@/lib/query-keys"; import type { InboundDocumentDto } from "@/lib/validations/inbound-documents"; import { DocumentLinkPicker } from "./document-link-picker"; +import { ImportFromSourceButton } from "./sources/import-from-source-button"; import { DOCUMENT_KIND_ICONS } from "./document-kind-meta"; import { documentDateKey } from "./vault-utils"; @@ -167,6 +168,9 @@ export function EpisodeDocumentsCard({ episodeId }: { episodeId: string }) { {t("documents.episodeCard.upload")} + ) : null} {hasMore ? ( diff --git a/src/components/documents/sources/__tests__/document-sources-ui.test.tsx b/src/components/documents/sources/__tests__/document-sources-ui.test.tsx new file mode 100644 index 000000000..2e3f93881 --- /dev/null +++ b/src/components/documents/sources/__tests__/document-sources-ui.test.tsx @@ -0,0 +1,230 @@ +/** + * The document picker's client gates (#1038), pinned via static renders: + * + * - the import button exists only for a person in their own record with a + * usable connection; a delegate, an unset operator list, no connection, or + * a connection whose origin was removed leave no button; + * - its label names the one connected system, or stays generic for two; + * - the settings card renders nothing unless the operator enabled the + * picker, lists the allowed origins, and never puts a token in the markup; + * - every error code the routes emit has its own sentence, and an unknown + * one falls back to the generic sentence rather than a raw key. + */ +import { describe, expect, it, vi } from "vitest"; +import { renderToStaticMarkup } from "react-dom/server"; +import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; + +let modules: Record = { inboundDocuments: true }; +vi.mock("@/hooks/use-auth", () => ({ + useAuth: () => ({ + user: { id: "u1", modules }, + isLoading: false, + isAuthenticated: true, + }), +})); + +let shared = false; +vi.mock("@/hooks/use-record-capabilities", () => ({ + useRecordCapabilities: () => ({ + inSharedRecord: shared, + canManageDomain: () => !shared, + }), +})); + +vi.mock("@/lib/api/api-fetch", async () => { + const actual = await vi.importActual( + "@/lib/api/api-fetch", + ); + return { ...actual, apiGet: () => new Promise(() => {}) }; +}); + +import { DocumentSourcesCard } from "@/components/settings/integrations/document-sources-card"; +import en from "../../../../../messages/en.json"; +import { ApiError } from "@/lib/api/api-fetch"; +import { + DOCUMENT_SOURCE_ERROR_CODES, + type DocumentSourcesStatusDto, +} from "@/lib/documents/sources/types"; +import { I18nProvider } from "@/lib/i18n/context"; +import { queryKeys } from "@/lib/query-keys"; + +import { ImportFromSourceButton } from "../import-from-source-button"; +import { mergeIds } from "../merge-ids"; +import { errorCodeOf, sourceErrorMessage } from "../source-errors"; + +const PAPERLESS = { + system: "PAPERLESS" as const, + baseUrl: "http://paperless.lan:8000", + organizationId: null, + hasToken: true as const, + lastVerifiedAt: "2026-09-20T10:00:00.000Z", + originAllowed: true, +}; +const PAPRA = { + ...PAPERLESS, + system: "PAPRA" as const, + baseUrl: "https://papra.example.com", + organizationId: "org_1", +}; + +function render(node: React.ReactNode, status?: DocumentSourcesStatusDto) { + const queryClient = new QueryClient({ + defaultOptions: { queries: { retry: false } }, + }); + if (status) + queryClient.setQueryData(queryKeys.documentSourcesStatus(), status); + return renderToStaticMarkup( + + {node} + , + ); +} + +const on = (connections: DocumentSourcesStatusDto["connections"]) => ({ + available: true, + allowedOrigins: ["http://paperless.lan:8000", "https://papra.example.com"], + connections, +}); + +describe("", () => { + it("names the one connected system", () => { + const html = render(, on([PAPERLESS])); + expect(html).toContain('data-slot="document-source-open"'); + expect(html).toContain("Import from Paperless-ngx"); + }); + + it("stays generic with two", () => { + const html = render(, on([PAPERLESS, PAPRA])); + expect(html).toContain("Import from archive"); + }); + + it("renders nothing without a usable connection", () => { + expect(render(, on([]))).toBe(""); + expect( + render( + , + on([{ ...PAPERLESS, originAllowed: false }]), + ), + ).toBe(""); + expect( + render(, { + available: false, + allowedOrigins: [], + connections: [], + }), + ).toBe(""); + }); + + it("renders nothing for a delegate, even with the owner's data cached", () => { + shared = true; + try { + expect(render(, on([PAPERLESS]))).toBe(""); + } finally { + shared = false; + } + }); + + it("renders nothing with the documents module off", () => { + modules = { inboundDocuments: false }; + try { + expect(render(, on([PAPERLESS]))).toBe(""); + } finally { + modules = { inboundDocuments: true }; + } + }); +}); + +describe("", () => { + it("renders nothing when the operator has not enabled the picker", () => { + expect( + render(, { + available: false, + allowedOrigins: [], + connections: [], + }), + ).toBe(""); + expect(render()).toBe(""); + }); + + it("renders nothing for a delegate", () => { + shared = true; + try { + expect(render(, on([PAPERLESS]))).toBe(""); + } finally { + shared = false; + } + }); + + it("shows the form, the allowed origins and a saved-token placeholder, never a token", () => { + const html = render(, on([PAPERLESS])); + expect(html).toContain('data-testid="document-sources-card"'); + expect(html).toContain("Document archives"); + expect(html).toContain( + "Allowed on this server: http://paperless.lan:8000, https://papra.example.com", + ); + expect(html).toContain("Saved. Leave empty to keep it."); + expect(html).toContain('value="http://paperless.lan:8000"'); + // The password field is empty: the server never sends the token back. + expect(html).toMatch(/id="document-source-token-paperless"[^>]*value=""/); + expect(html).toContain("Disconnect"); + }); + + it("offers no disconnect or test for a system that is not connected", () => { + const html = render(, on([])); + expect(html).toContain("Not connected"); + expect(html).not.toContain("Disconnect"); + expect(html).toContain("Save and test"); + }); +}); + +describe("source error sentences", () => { + const t = (key: string) => { + let node: unknown = en; + for (const part of key.split(".")) { + node = (node as Record | undefined)?.[part]; + } + return typeof node === "string" ? node : key; + }; + const generic = t("documents.sourcePicker.errors.generic"); + + it("has its own sentence for every code the routes emit", () => { + const codes = [ + ...DOCUMENT_SOURCE_ERROR_CODES, + "documents.sources.browserOnly", + "documents.sources.invalidAddress", + "documents.sources.organizationRequired", + "documents.sources.tokenRequired", + "documents.inbound.fileTooLarge", + "documents.inbound.quotaExceeded", + "documents.inbound.fileType", + "documents.inbound.rateLimited", + "documents.inbound.uploadBusy", + "module.disabled", + ]; + const sentences = new Set(); + for (const code of codes) { + const sentence = sourceErrorMessage(t, code); + expect(sentence, code).not.toBe(generic); + expect(sentence, code).not.toContain("documents.sourcePicker"); + sentences.add(sentence); + } + expect(sentences.size).toBe(codes.length); + expect(sourceErrorMessage(t, "something.else")).toBe(generic); + expect(sourceErrorMessage(t, undefined)).toBe(generic); + }); + + it("reads the code off a failed request", () => { + expect( + errorCodeOf( + new ApiError("x", 502, { errorCode: "documents.sources.authRefused" }), + ), + ).toBe("documents.sources.authRefused"); + expect(errorCodeOf(new Error("x"))).toBeUndefined(); + }); +}); + +describe("mergeIds", () => { + it("adds imported ids to a form's selection once, in order", () => { + expect(mergeIds(["a", "b"], ["b", "c", "c"])).toEqual(["a", "b", "c"]); + }); +}); diff --git a/src/components/documents/sources/document-source-picker.tsx b/src/components/documents/sources/document-source-picker.tsx new file mode 100644 index 000000000..2199193dc --- /dev/null +++ b/src/components/documents/sources/document-source-picker.tsx @@ -0,0 +1,654 @@ +"use client"; + +/** + * The document picker (#1038): search a connected Paperless-ngx or Papra by + * document name, narrow by tag and date, tick the documents to bring over, + * import them. + * + * Name first, because that is how the person looks for a letter: the search + * field opens focused and debounces into the source's own name search. Tag and + * date are filters beside it, not the entry point. Each result says what + * HealthLog already holds under that source key: already imported, deleted + * here (not offered again: a deletion was a decision), or new. + * + * The import runs one document per request, in order, so every row can show + * its own progress and result, and one failure does not hide the others. A + * refusal that will answer every following document the same way (the hourly + * upload allowance, a full vault, a refused token) stops the run and says so. + * + * Where it was opened from decides what an import does besides storing: + * + * - `link`: the record already exists (a condition's page), so the server + * files the document against it as part of the import; + * - `onImported`: a form that owns its selection (a visit, a vaccination) + * gets the new ids and adds them itself. Linking on the server there would + * be undone by the form's own save, which replaces its links. + * + * With either, a document already in HealthLog can be picked too: it is not + * stored twice, it is linked or handed to the form. + */ +import { + useInfiniteQuery, + useQuery, + useQueryClient, +} from "@tanstack/react-query"; +import { + Check, + CircleAlert, + CircleCheck, + Loader2, + Search, + X, +} from "lucide-react"; +import { useEffect, useMemo, useRef, useState } from "react"; + +import { Badge } from "@/components/ui/badge"; +import { Button } from "@/components/ui/button"; +import { DateField } from "@/components/ui/date-field"; +import { Input } from "@/components/ui/input"; +import { Label } from "@/components/ui/label"; +import { NativeSelect } from "@/components/ui/native-select"; +import { QueryErrorRow } from "@/components/ui/query-error-row"; +import { ResponsiveSheet } from "@/components/ui/responsive-sheet"; +import { Skeleton } from "@/components/ui/skeleton"; +import { TagChip } from "@/components/ui/tag-chip"; +import { apiGet, apiPost } from "@/lib/api/api-fetch"; +import { + DOCUMENT_PICKER_MAX_SELECTION, + slugForSystem, + type DocumentImportOutcome, + type DocumentPickerLinkKind, + type DocumentPickerSystem, + type DocumentSourceImportDto, + type DocumentSourceResultDto, + type DocumentSourceSearchDto, + type DocumentSourceTagDto, +} from "@/lib/documents/sources/types"; +import { useFormatters, useTranslations } from "@/lib/i18n/context"; +import { invalidateKeys, queryKeys } from "@/lib/query-keys"; +import { cn } from "@/lib/utils"; +import type { InboundDocumentKindValue } from "@/lib/validations/inbound-documents"; + +import { formatBytes } from "../vault-utils"; +import { + errorCodeOf, + isStoppingError, + sourceErrorMessage, +} from "./source-errors"; +import { systemName } from "./use-document-sources"; + +export interface DocumentPickerLink { + kind: DocumentPickerLinkKind; + id: string; +} + +export interface DocumentSourcePickerProps { + open: boolean; + onOpenChange: (open: boolean) => void; + /** Connected systems, in the order the switch shows them. */ + systems: DocumentPickerSystem[]; + /** File the imported documents against this existing record. */ + link?: DocumentPickerLink; + /** Document type for everything imported (default OTHER). */ + kind?: InboundDocumentKindValue; + /** Hand the vault ids to a form that links them on its own save. */ + onImported?: (documentIds: string[]) => void; +} + +type RowRun = + | { state: "waiting" } + | { state: "working" } + | { state: DocumentImportOutcome; linked: boolean } + | { state: "failed"; message: string }; + +const SEARCH_DEBOUNCE_MS = 300; +const IMPORT_TIMEOUT_MS = 120_000; + +export function DocumentSourcePicker({ + open, + onOpenChange, + systems, + link, + kind, + onImported, +}: DocumentSourcePickerProps) { + const { t, locale } = useTranslations(); + const format = useFormatters(); + const queryClient = useQueryClient(); + + const [system, setSystem] = useState(null); + const active = + system && systems.includes(system) ? system : (systems[0] ?? null); + const slug = active ? slugForSystem(active) : null; + + const [searchDraft, setSearchDraft] = useState(""); + const [q, setQ] = useState(""); + const [tag, setTag] = useState(""); + const [from, setFrom] = useState(""); + const [to, setTo] = useState(""); + const [selected, setSelected] = useState>(new Map()); + const [runs, setRuns] = useState>(new Map()); + const [running, setRunning] = useState(false); + const [finished, setFinished] = useState(false); + const [stopMessage, setStopMessage] = useState(null); + + const debounceRef = useRef | null>(null); + useEffect( + () => () => { + if (debounceRef.current) clearTimeout(debounceRef.current); + }, + [], + ); + const onSearchChange = (value: string) => { + setSearchDraft(value); + if (debounceRef.current) clearTimeout(debounceRef.current); + debounceRef.current = setTimeout( + () => setQ(value.trim()), + SEARCH_DEBOUNCE_MS, + ); + }; + + const reset = () => { + setSearchDraft(""); + setQ(""); + setTag(""); + setFrom(""); + setTo(""); + setSelected(new Map()); + setRuns(new Map()); + setFinished(false); + setStopMessage(null); + }; + + const switchSystem = (next: DocumentPickerSystem) => { + if (next === active || running) return; + reset(); + setSystem(next); + }; + + const tags = useQuery({ + queryKey: queryKeys.documentSourceTags(slug ?? ""), + enabled: open && slug !== null, + queryFn: () => + apiGet<{ tags: DocumentSourceTagDto[] }>( + `/api/documents/sources/${slug}/tags`, + ), + staleTime: 5 * 60_000, + }); + + const facets = { + q, + ...(tag ? { tag } : {}), + ...(from ? { from } : {}), + ...(to ? { to } : {}), + }; + const rangeInvalid = Boolean(from && to && from > to); + const search = useInfiniteQuery({ + queryKey: queryKeys.documentSourceSearch(slug ?? "", facets), + enabled: open && slug !== null && !rangeInvalid, + initialPageParam: 1, + queryFn: ({ pageParam }) => { + const sp = new URLSearchParams({ page: String(pageParam) }); + if (q) sp.set("q", q); + if (tag) sp.set("tag", tag); + if (from) sp.set("from", from); + if (to) sp.set("to", to); + return apiGet( + `/api/documents/sources/${slug}/search?${sp.toString()}`, + ); + }, + getNextPageParam: (last) => (last.hasMore ? last.page + 1 : undefined), + staleTime: 30_000, + }); + + const results = useMemo(() => { + const seen = new Set(); + const out: DocumentSourceResultDto[] = []; + for (const page of search.data?.pages ?? []) { + for (const row of page.results) { + if (seen.has(row.sourceId)) continue; + seen.add(row.sourceId); + out.push(row); + } + } + return out; + }, [search.data]); + + // An already imported document is worth picking only when the pick does + // something with it: link it, or hand it to the form. + const canTakeImported = Boolean(link || onImported); + const selectable = (row: DocumentSourceResultDto) => + row.state === "new" || (row.state === "imported" && canTakeImported); + + const toggle = (row: DocumentSourceResultDto) => { + if (running || finished || !selectable(row)) return; + setSelected((prev) => { + const next = new Map(prev); + if (next.has(row.sourceId)) next.delete(row.sourceId); + else if (next.size < DOCUMENT_PICKER_MAX_SELECTION) { + next.set(row.sourceId, row.title); + } + return next; + }); + }; + + async function runImport() { + if (!slug || selected.size === 0) return; + setRunning(true); + setStopMessage(null); + const order = [...selected.keys()]; + setRuns(new Map(order.map((id) => [id, { state: "waiting" }]))); + const handed: string[] = []; + const mark = (id: string, run: RowRun) => + setRuns((prev) => new Map(prev).set(id, run)); + + for (const sourceId of order) { + mark(sourceId, { state: "working" }); + try { + const result = await apiPost( + `/api/documents/sources/${slug}/import`, + { + sourceId, + ...(kind ? { kind } : {}), + ...(link ? { link } : {}), + }, + // The server may take up to a minute to fetch a large file from + // the source; the default 15 s client timeout would abandon it. + { signal: AbortSignal.timeout(IMPORT_TIMEOUT_MS) }, + ); + mark(sourceId, { state: result.outcome, linked: result.linked }); + if (result.outcome !== "deleted" && result.documentId) { + handed.push(result.documentId); + } + } catch (err) { + const code = errorCodeOf(err); + const message = sourceErrorMessage(t, code); + mark(sourceId, { state: "failed", message }); + if (isStoppingError(code)) { + setStopMessage(message); + break; + } + } + } + + setRunning(false); + setFinished(true); + setSelected(new Map()); + // The vault, its pickers and the record's own lists read documents + // through the `["documents"]` prefix; the picker's states through its own. + void invalidateKeys(queryClient, [ + queryKeys.documents(), + queryKeys.documentSources(), + ]); + if (handed.length > 0) onImported?.(handed); + } + + const counts = useMemo(() => { + let imported = 0; + let present = 0; + let failed = 0; + for (const run of runs.values()) { + if (run.state === "imported") imported += 1; + else if (run.state === "duplicate" || run.state === "deleted") + present += 1; + else if (run.state === "failed") failed += 1; + } + return { imported, present, failed }; + }, [runs]); + + const name = active ? systemName(active) : ""; + const hasFilters = Boolean(tag || from || to); + + const footer = ( +
+

+ {finished + ? t("documents.sourcePicker.summary", counts) + : running + ? t("documents.sourcePicker.importing") + : t("documents.sourcePicker.selectedCount", { + count: selected.size, + max: DOCUMENT_PICKER_MAX_SELECTION, + })} +

+ {finished ? ( + + ) : ( + + )} +
+ ); + + return ( + { + // A run in progress finishes before the sheet can close; its rows + // are the only place the outcome is shown. + if (!next && running) return; + if (!next) reset(); + onOpenChange(next); + }} + title={t("documents.sourcePicker.title", { name })} + description={t("documents.sourcePicker.description")} + contentWidth="lg" + footer={footer} + > +
+ {systems.length > 1 ? ( +
+ {systems.map((option) => ( + + ))} +
+ ) : null} + +
+ +
+ + onSearchChange(e.target.value)} + placeholder={t("documents.sourcePicker.searchPlaceholder")} + className="pl-9" + maxLength={200} + autoComplete="off" + data-slot="document-source-search" + /> +
+
+ +
+
+ + setTag(e.target.value)} + disabled={tags.isPending && !tags.isError} + > + + {(tags.data?.tags ?? []).map((option) => ( + + ))} + +
+
+ + +
+
+ + +
+
+ {hasFilters ? ( + + ) : null} + + {stopMessage ? ( +

+ {t("documents.sourcePicker.stopped", { reason: stopMessage })} +

+ ) : null} + + {rangeInvalid ? ( +

+ {t("documents.sourcePicker.rangeInvalid")} +

+ ) : search.isPending ? ( +
+ {Array.from({ length: 4 }, (_, i) => ( + + ))} +
+ ) : search.isError ? ( + void search.refetch()} + slot="document-source-error" + /> + ) : results.length === 0 ? ( +

+ {q || hasFilters + ? t("documents.sourcePicker.noMatch") + : t("documents.sourcePicker.empty", { name })} +

+ ) : ( +
    + {results.map((row) => { + const on = selected.has(row.sourceId); + const run = runs.get(row.sourceId); + const allowed = selectable(row); + const meta = [ + row.date ? format.date(`${row.date}T12:00:00.000Z`) : null, + row.sizeBytes !== null + ? formatBytes(row.sizeBytes, locale) + : null, + ] + .filter(Boolean) + .join(" · "); + return ( +
  • + +
    • + ); + })} +
+ )} + + {search.hasNextPage && !search.isError ? ( + + ) : null} +
+ + ); +} + +/** One row's progress or result, under its title. */ +function RunLine({ run }: { run: RowRun }) { + const { t } = useTranslations(); + switch (run.state) { + case "waiting": + return ( + + {t("documents.sourcePicker.result.waiting")} + + ); + case "working": + return ( + + + {t("documents.sourcePicker.result.working")} + + ); + case "failed": + return ( + + + {run.message} + + ); + case "deleted": + return ( + + {t("documents.sourcePicker.result.deleted")} + + ); + case "imported": + case "duplicate": + return ( + + + {run.state === "imported" + ? run.linked + ? t("documents.sourcePicker.result.importedLinked") + : t("documents.sourcePicker.result.imported") + : run.linked + ? t("documents.sourcePicker.result.duplicateLinked") + : t("documents.sourcePicker.result.duplicate")} + + ); + } +} diff --git a/src/components/documents/sources/import-from-source-button.tsx b/src/components/documents/sources/import-from-source-button.tsx new file mode 100644 index 000000000..09fd872df --- /dev/null +++ b/src/components/documents/sources/import-from-source-button.tsx @@ -0,0 +1,73 @@ +"use client"; + +/** + * "Import from Paperless-ngx / Papra": the button that opens the document + * picker (#1038), for every place a document can be added or linked. + * + * Renders nothing unless the person, in their own record, has at least one + * usable connection: no connection, a delegate or guardian, the module off, or + * the operator's list unset all leave no trace, because a button that can only + * lead to a refusal advertises a feature that is not there. Connecting happens + * in Settings, not here. + */ +import { Import } from "lucide-react"; +import { useState } from "react"; + +import { Button } from "@/components/ui/button"; +import { useTranslations } from "@/lib/i18n/context"; +import type { InboundDocumentKindValue } from "@/lib/validations/inbound-documents"; + +import { + DocumentSourcePicker, + type DocumentPickerLink, +} from "./document-source-picker"; +import { systemName, useDocumentSourcesStatus } from "./use-document-sources"; + +export function ImportFromSourceButton({ + link, + kind, + onImported, + className, +}: { + link?: DocumentPickerLink; + kind?: InboundDocumentKindValue; + onImported?: (documentIds: string[]) => void; + className?: string; +}) { + const { t } = useTranslations(); + const { connected } = useDocumentSourcesStatus(); + const [open, setOpen] = useState(false); + + if (connected.length === 0) return null; + + const label = + connected.length === 1 + ? t("documents.sourcePicker.openNamed", { + name: systemName(connected[0]), + }) + : t("documents.sourcePicker.open"); + + return ( + <> + + + + ); +} diff --git a/src/components/documents/sources/merge-ids.ts b/src/components/documents/sources/merge-ids.ts new file mode 100644 index 000000000..ab8579349 --- /dev/null +++ b/src/components/documents/sources/merge-ids.ts @@ -0,0 +1,6 @@ +/** Add imported ids to a form's selection, in order, never twice. */ +export function mergeIds(selected: string[], added: string[]): string[] { + const merged = [...selected]; + for (const id of added) if (!merged.includes(id)) merged.push(id); + return merged; +} diff --git a/src/components/documents/sources/source-errors.ts b/src/components/documents/sources/source-errors.ts new file mode 100644 index 000000000..3c1739924 --- /dev/null +++ b/src/components/documents/sources/source-errors.ts @@ -0,0 +1,81 @@ +/** + * The document picker's error codes (#1038), as sentences. One place for the + * sheet and the settings card, so the same refusal reads the same on both. + * Every branch names its key literally so the i18n coverage guard can see it. + */ +import { ApiError } from "@/lib/api/api-fetch"; + +type T = (key: string, params?: Record) => string; + +/** The `meta.errorCode` a failed request carried, if any. */ +export function errorCodeOf(err: unknown): string | undefined { + if (err instanceof ApiError) { + const code = err.meta?.errorCode; + if (typeof code === "string") return code; + } + return undefined; +} + +/** Is this a refusal that will not change on the next document? */ +export function isStoppingError(code: string | undefined): boolean { + return ( + code === "documents.inbound.rateLimited" || + code === "documents.inbound.quotaExceeded" || + code === "documents.sources.authRefused" || + code === "documents.sources.unreachable" || + code === "documents.sources.originNotAllowed" || + code === "documents.sources.notConnected" || + code === "documents.sources.unavailable" || + code === "documents.sources.browserOnly" || + code === "documents.sources.linkTargetNotFound" + ); +} + +export function sourceErrorMessage(t: T, code: string | undefined): string { + switch (code) { + case "documents.sources.unavailable": + return t("documents.sourcePicker.errors.unavailable"); + case "documents.sources.notConnected": + return t("documents.sourcePicker.errors.notConnected"); + case "documents.sources.originNotAllowed": + return t("documents.sourcePicker.errors.originNotAllowed"); + case "documents.sources.unreachable": + return t("documents.sourcePicker.errors.unreachable"); + case "documents.sources.redirected": + return t("documents.sourcePicker.errors.redirected"); + case "documents.sources.authRefused": + return t("documents.sourcePicker.errors.authRefused"); + case "documents.sources.versionTooOld": + return t("documents.sourcePicker.errors.versionTooOld"); + case "documents.sources.badResponse": + return t("documents.sourcePicker.errors.badResponse"); + case "documents.sources.notFound": + return t("documents.sourcePicker.errors.notFound"); + case "documents.sources.linkTargetNotFound": + return t("documents.sourcePicker.errors.linkTargetNotFound"); + case "documents.sources.rateLimited": + return t("documents.sourcePicker.errors.rateLimited"); + case "documents.sources.browserOnly": + return t("documents.sourcePicker.errors.browserOnly"); + case "documents.sources.invalidAddress": + return t("documents.sourcePicker.errors.invalidAddress"); + case "documents.sources.organizationRequired": + return t("documents.sourcePicker.errors.organizationRequired"); + case "documents.sources.tokenRequired": + return t("documents.sourcePicker.errors.tokenRequired"); + case "documents.inbound.fileTooLarge": + return t("documents.sourcePicker.errors.fileTooLarge"); + case "documents.inbound.quotaExceeded": + return t("documents.sourcePicker.errors.quotaExceeded"); + case "documents.inbound.fileType": + return t("documents.sourcePicker.errors.fileType"); + case "documents.inbound.rateLimited": + return t("documents.sourcePicker.errors.uploadLimit"); + case "documents.inbound.uploadBusy": + return t("documents.sourcePicker.errors.uploadBusy"); + case "module.disabled": + return t("documents.sourcePicker.errors.moduleDisabled"); + default: + return t("documents.sourcePicker.errors.generic"); + } +} diff --git a/src/components/documents/sources/use-document-sources.ts b/src/components/documents/sources/use-document-sources.ts new file mode 100644 index 000000000..13b1a4692 --- /dev/null +++ b/src/components/documents/sources/use-document-sources.ts @@ -0,0 +1,51 @@ +"use client"; + +/** + * The document picker's availability for the person on this screen (#1038). + * + * Reads `GET /api/documents/sources` only when it can matter: the documents + * module is on and the person is in their own record with the right to manage + * documents. A delegate or guardian never asks (the route would refuse them + * anyway), so the import buttons and the settings card render nothing for + * them. `connected` is the list of systems with a usable connection, in a + * stable order; an entry whose origin the operator removed is left out. + */ +import { useQuery } from "@tanstack/react-query"; + +import { useAuth } from "@/hooks/use-auth"; +import { useRecordCapabilities } from "@/hooks/use-record-capabilities"; +import { apiGet } from "@/lib/api/api-fetch"; +import type { + DocumentPickerSystem, + DocumentSourcesStatusDto, +} from "@/lib/documents/sources/types"; +import { queryKeys } from "@/lib/query-keys"; + +export function useDocumentSourcesStatus() { + const { user } = useAuth(); + const { canManageDomain, inSharedRecord } = useRecordCapabilities(); + const eligible = + user?.modules?.inboundDocuments === true && + !inSharedRecord && + canManageDomain("documents"); + + const query = useQuery({ + queryKey: queryKeys.documentSourcesStatus(), + enabled: eligible, + queryFn: () => apiGet("/api/documents/sources"), + staleTime: 60_000, + }); + + const connected: DocumentPickerSystem[] = eligible + ? (query.data?.connections ?? []) + .filter((c) => c.originAllowed) + .map((c) => c.system) + : []; + + return { eligible, query, connected }; +} + +/** Display name of a system; product names, not translated. */ +export function systemName(system: DocumentPickerSystem): string { + return system === "PAPERLESS" ? "Paperless-ngx" : "Papra"; +} diff --git a/src/components/encounters/encounter-link-pickers.tsx b/src/components/encounters/encounter-link-pickers.tsx index 1dcf1e536..1e0eba0fe 100644 --- a/src/components/encounters/encounter-link-pickers.tsx +++ b/src/components/encounters/encounter-link-pickers.tsx @@ -32,6 +32,8 @@ import { type EntityLinkOption, } from "@/components/links/entity-link-picker"; import { useVaultDocumentOptions } from "@/components/links/vault-document-options"; +import { ImportFromSourceButton } from "@/components/documents/sources/import-from-source-button"; +import { mergeIds } from "@/components/documents/sources/merge-ids"; /** Fetch cap for the picker — the grouped sheet needs enough rows to group. */ export const PICKER_FETCH_LIMIT = 200; @@ -152,6 +154,17 @@ export function EncounterLinkPickers({ options={documents.options} searchPlaceholder={t("links.picker.searchPlaceholder")} emptyLabel={t("encounters.form.linkNothingToOffer")} + extraAction={ + // The form owns its selection and replaces the visit's links on + // save, so imported documents join the selection rather than + // being linked on the server. + + onChange({ documentIds: mergeIds(documentIds, ids) }) + } + /> + } /> ) : null} diff --git a/src/components/links/entity-link-picker.tsx b/src/components/links/entity-link-picker.tsx index 3b3f4a467..8d70d10b0 100644 --- a/src/components/links/entity-link-picker.tsx +++ b/src/components/links/entity-link-picker.tsx @@ -31,7 +31,7 @@ * carries the option's date, so two doses of one vaccine read as two * different records. */ -import { useMemo, useState } from "react"; +import { useMemo, useState, type ReactNode } from "react"; import Link from "next/link"; import { Check, Plus, Search, X } from "lucide-react"; import type { LucideIcon } from "lucide-react"; @@ -164,6 +164,7 @@ export function EntityLinkPicker({ error = false, errorLabel, onRetry, + extraAction, }: { icon: LucideIcon; title: string; @@ -183,6 +184,13 @@ export function EntityLinkPicker({ error?: boolean; errorLabel?: string; onRetry?: () => void; + /** + * A second way to add, beside the add button: the documents pickers put + * "Import from Paperless-ngx / Papra" here. Shown even when there is + * nothing to link yet, because an empty vault is exactly when importing is + * the way forward. The element renders nothing when it has nothing to offer. + */ + extraAction?: ReactNode; }) { const { t } = useTranslations(); const [open, setOpen] = useState(false); @@ -238,7 +246,10 @@ export function EntityLinkPicker({ slot={`${slot}-error`} /> ) : options.length === 0 ? ( -

{emptyLabel}

+
+

{emptyLabel}

+ {extraAction} +
) : (
{selectedChips.length > 0 ? ( @@ -312,17 +323,20 @@ export function EntityLinkPicker({

) : null} - +
+ + {extraAction} +
)} diff --git a/src/components/settings/integrations-section.tsx b/src/components/settings/integrations-section.tsx index 37ceed391..4b6ce1506 100644 --- a/src/components/settings/integrations-section.tsx +++ b/src/components/settings/integrations-section.tsx @@ -37,6 +37,7 @@ import { ConnectionsPanel } from "@/components/settings/integrations/connections import { SectionHeading } from "@/components/ui/section-heading"; import { NotificationChannelsPanel } from "@/components/settings/integrations/notification-channels-panel"; import { NutrientIntakeCard } from "@/components/settings/nutrient-intake-card"; +import { DocumentSourcesCard } from "@/components/settings/integrations/document-sources-card"; import { Button } from "@/components/ui/button"; export { @@ -77,6 +78,11 @@ export function IntegrationsSection({ under the same Connections group rather than a new labelled section that could paint an orphaned heading over nothing. */} + + {/* v1.39.3 (#1038) — Paperless-ngx / Papra for the document picker. + Renders nothing unless the operator listed an origin in + DOCUMENT_SOURCE_ORIGINS and the person is in their own record. */} + {/* Delivery channels ("where it's delivered"). The existing channels diff --git a/src/components/settings/integrations/document-sources-card.tsx b/src/components/settings/integrations/document-sources-card.tsx new file mode 100644 index 000000000..b198d16ec --- /dev/null +++ b/src/components/settings/integrations/document-sources-card.tsx @@ -0,0 +1,415 @@ +"use client"; + +/** + * Settings → Integrations → Document archives (#1038): connect Paperless-ngx or + * Papra so the document picker can search it. + * + * Renders nothing unless the operator listed at least one origin in + * `DOCUMENT_SOURCE_ORIGINS`, the documents module is on, and the person is in + * their own record: a delegate never manages the owner's connection, and a + * card that can only be refused advertises a feature that is not there. + * + * One form per system behind a switch. Save tests the connection before it is + * stored and says why when it fails; the token field never shows the stored + * token (the server never returns it), only that one is saved, and leaving it + * empty keeps it. Results sit above the action row; nothing follows it + * (UI standards §12). + */ +import { useMutation, useQueryClient } from "@tanstack/react-query"; +import { Archive, CheckCircle2, Loader2, Unlink } from "lucide-react"; +import { useState } from "react"; + +import { + AlertDialog, + AlertDialogAction, + AlertDialogCancel, + AlertDialogContent, + AlertDialogDescription, + AlertDialogFooter, + AlertDialogHeader, + AlertDialogTitle, + AlertDialogTrigger, +} from "@/components/ui/alert-dialog"; +import { Badge } from "@/components/ui/badge"; +import { Button } from "@/components/ui/button"; +import { Input } from "@/components/ui/input"; +import { Label } from "@/components/ui/label"; +import { PasswordInput } from "@/components/ui/password-input"; +import { SettingsCardActions } from "@/components/settings/_card-actions"; +import { SettingsCardHeader } from "@/components/settings/_card-header"; +import { SettingsCard } from "@/components/settings/settings-card"; +import { + errorCodeOf, + sourceErrorMessage, +} from "@/components/documents/sources/source-errors"; +import { + systemName, + useDocumentSourcesStatus, +} from "@/components/documents/sources/use-document-sources"; +import { apiDelete, apiPost, apiPut } from "@/lib/api/api-fetch"; +import { + DOCUMENT_PICKER_SYSTEMS, + slugForSystem, + type DocumentPickerSystem, + type DocumentSourceConnectionDto, +} from "@/lib/documents/sources/types"; +import { useFormatters, useTranslations } from "@/lib/i18n/context"; +import { invalidateKeys, queryKeys } from "@/lib/query-keys"; +import { cn } from "@/lib/utils"; + +type Outcome = { tone: "success" | "error"; message: string } | null; + +export function DocumentSourcesCard() { + const { t } = useTranslations(); + const { eligible, query } = useDocumentSourcesStatus(); + const [system, setSystem] = useState("PAPERLESS"); + + if (!eligible || !query.data?.available) return null; + + const connections = query.data.connections; + const connectedCount = connections.length; + + return ( + + 0 ? ( + + {t("settings.documentSources.statusConnected")} + + ) : null + } + /> + +

{t("settings.documentSources.explainer")}

+ +
+ {DOCUMENT_PICKER_SYSTEMS.map((option) => { + const isConnected = connections.some((c) => c.system === option); + return ( + + ); + })} +
+ + c.system === system) ?? null} + allowedOrigins={query.data.allowedOrigins} + /> + + ); +} + +function SourceForm({ + system, + connection, + allowedOrigins, +}: { + system: DocumentPickerSystem; + connection: DocumentSourceConnectionDto | null; + allowedOrigins: string[]; +}) { + const { t } = useTranslations(); + const format = useFormatters(); + const queryClient = useQueryClient(); + const slug = slugForSystem(system); + const isPapra = system === "PAPRA"; + + const [baseUrl, setBaseUrl] = useState(connection?.baseUrl ?? ""); + const [organizationId, setOrganizationId] = useState( + connection?.organizationId ?? "", + ); + const [token, setToken] = useState(""); + const [outcome, setOutcome] = useState(null); + + const dirty = + baseUrl.trim() !== (connection?.baseUrl ?? "") || + (isPapra && organizationId.trim() !== (connection?.organizationId ?? "")) || + token.trim() !== ""; + + const refresh = () => + invalidateKeys(queryClient, [queryKeys.documentSources()]); + + const save = useMutation({ + mutationFn: () => + apiPut(`/api/documents/sources/${slug}`, { + baseUrl: baseUrl.trim(), + ...(isPapra ? { organizationId: organizationId.trim() } : {}), + ...(token.trim() ? { token: token.trim() } : {}), + }), + onMutate: () => setOutcome(null), + onSuccess: () => { + setToken(""); + setOutcome({ + tone: "success", + message: t("settings.documentSources.saved"), + }); + void refresh(); + }, + onError: (err) => + setOutcome({ + tone: "error", + message: sourceErrorMessage(t, errorCodeOf(err)), + }), + }); + + const test = useMutation({ + mutationFn: () => + apiPost<{ ok: true; latencyMs: number }>( + `/api/documents/sources/${slug}/test`, + ), + onMutate: () => setOutcome(null), + onSuccess: (data) => { + setOutcome({ + tone: "success", + message: t("settings.documentSources.testOk", { + latency: data.latencyMs, + }), + }); + void refresh(); + }, + onError: (err) => + setOutcome({ + tone: "error", + message: sourceErrorMessage(t, errorCodeOf(err)), + }), + }); + + const disconnect = useMutation({ + mutationFn: () => apiDelete(`/api/documents/sources/${slug}`), + onSuccess: () => { + setBaseUrl(""); + setOrganizationId(""); + setToken(""); + setOutcome(null); + void refresh(); + }, + onError: (err) => + setOutcome({ + tone: "error", + message: sourceErrorMessage(t, errorCodeOf(err)), + }), + }); + + const busy = save.isPending || test.isPending || disconnect.isPending; + const canSave = + baseUrl.trim() !== "" && + (!isPapra || organizationId.trim() !== "") && + (connection !== null || token.trim() !== "") && + dirty; + + return ( +
{ + e.preventDefault(); + if (canSave && !busy) save.mutate(); + }} + data-slot="document-sources-form" + > +

+ {connection + ? connection.lastVerifiedAt + ? t("settings.documentSources.connectedSince", { + date: format.date(connection.lastVerifiedAt), + }) + : t("settings.documentSources.statusConnected") + : t("settings.documentSources.statusNotConnected")} +

+ + {connection && !connection.originAllowed ? ( +

+ {t("settings.documentSources.originRemoved")} +

+ ) : null} + +
+ + setBaseUrl(e.target.value)} + placeholder={allowedOrigins[0] ?? "https://"} + maxLength={2048} + autoComplete="off" + inputMode="url" + spellCheck={false} + autoCapitalize="none" + /> +

+ {t("settings.documentSources.allowedOrigins", { + origins: allowedOrigins.join(", "), + })} +

+
+ + {isPapra ? ( +
+ + setOrganizationId(e.target.value)} + maxLength={128} + autoComplete="off" + spellCheck={false} + autoCapitalize="none" + /> +

+ {t("settings.documentSources.organizationHelp")} +

+
+ ) : null} + +
+ + setToken(e.target.value)} + placeholder={ + connection ? t("settings.documentSources.tokenSaved") : undefined + } + maxLength={512} + autoComplete="off" + spellCheck={false} + autoCapitalize="none" + /> +

+ {isPapra + ? t("settings.documentSources.tokenHelpPapra") + : t("settings.documentSources.tokenHelpPaperless")} +

+
+ + {outcome ? ( +

+ {outcome.message} +

+ ) : null} + + + {connection ? ( + + + + + + + + {t("settings.documentSources.disconnectTitle", { + name: systemName(system), + })} + + + {t("settings.documentSources.disconnectDescription")} + + + + {t("common.cancel")} + disconnect.mutate()}> + {t("settings.documentSources.disconnect")} + + + + + ) : null} + {connection ? ( + + ) : null} + + +
+ ); +} diff --git a/src/components/vaccinations/vaccination-document-picker.tsx b/src/components/vaccinations/vaccination-document-picker.tsx index d1778773b..6fb195261 100644 --- a/src/components/vaccinations/vaccination-document-picker.tsx +++ b/src/components/vaccinations/vaccination-document-picker.tsx @@ -28,6 +28,8 @@ import { FolderOpen } from "lucide-react"; import { useTranslations } from "@/lib/i18n/context"; import { EntityLinkPicker } from "@/components/links/entity-link-picker"; import { useVaultDocumentOptions } from "@/components/links/vault-document-options"; +import { ImportFromSourceButton } from "@/components/documents/sources/import-from-source-button"; +import { mergeIds } from "@/components/documents/sources/merge-ids"; export function VaccinationDocumentPicker({ enabled, @@ -73,6 +75,14 @@ export function VaccinationDocumentPicker({ options={vault.options} searchPlaceholder={t("links.picker.searchPlaceholder")} emptyLabel={t("vaccinations.form.linkNothingToOffer")} + extraAction={ + // Joins the form's selection; the dose's save writes the links. + onChange(mergeIds(documentIds, ids))} + /> + } /> ); diff --git a/src/lib/query-keys/documents.ts b/src/lib/query-keys/documents.ts index 2a407a2dc..c5c8e8dc9 100644 --- a/src/lib/query-keys/documents.ts +++ b/src/lib/query-keys/documents.ts @@ -81,4 +81,27 @@ export const documentKeys = { * toggle; a flip invalidates the document AI capability probe. */ documentsAutoAiRead: () => ["documents", "auto-ai-read"] as const, + /** + * The document picker (#1038): its own top-level prefix, deliberately NOT + * under `["documents"]`. Every vault write invalidates that prefix, and a + * search or tag read here is a request to the person's Paperless-ngx or + * Papra instance; a filing action in the vault must not re-send them. + */ + documentSources: () => ["documentSources"] as const, + /** Availability + the person's connections (`GET /api/documents/sources`). */ + documentSourcesStatus: () => ["documentSources", "status"] as const, + /** One source's tags for the filter. */ + documentSourceTags: (system: string) => + ["documentSources", "tags", system] as const, + /** One page of one source's search. */ + documentSourceSearch: (system: string, search: DocumentSourceSearchKey) => + ["documentSources", "search", system, search] as const, }; + +/** The facets that make one picker search distinct from another. */ +export interface DocumentSourceSearchKey { + q: string; + tag?: string; + from?: string; + to?: string; +} From 9e38ee3a99a303e55cee7017be36141e1e4e022b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Sat, 26 Sep 2026 13:32:52 +0200 Subject: [PATCH 08/18] fix(documents): report picker and card results through the shared outcome line, and register the disconnect as a destructive control --- .../destructive-control-guard.test.ts | 7 ++++++ .../success-affordance-guard.test.ts | 7 ++++++ .../sources/document-source-picker.tsx | 24 ++++++++++++++++++- .../integrations/document-sources-card.tsx | 22 +++++++---------- 4 files changed, 46 insertions(+), 14 deletions(-) diff --git a/src/__tests__/destructive-control-guard.test.ts b/src/__tests__/destructive-control-guard.test.ts index 27a745581..fcf209cfe 100644 --- a/src/__tests__/destructive-control-guard.test.ts +++ b/src/__tests__/destructive-control-guard.test.ts @@ -579,6 +579,13 @@ const REGISTRY: DestructiveEntry[] = [ recovery: "permanent", confirm: ["AlertDialog"], })), + { + file: "components/settings/integrations/document-sources-card.tsx", + destroys: + "the stored Paperless-ngx or Papra address and API token; documents already imported stay in the vault — reconnecting means typing the token again", + recovery: "permanent", + confirm: ["AlertDialog"], + }, ]; // ── Detectors ───────────────────────────────────────────────────────────── diff --git a/src/__tests__/success-affordance-guard.test.ts b/src/__tests__/success-affordance-guard.test.ts index 8cd324ece..d8f561058 100644 --- a/src/__tests__/success-affordance-guard.test.ts +++ b/src/__tests__/success-affordance-guard.test.ts @@ -161,6 +161,13 @@ const PINNED_AFFORDANCES: Record< "src/components/documents/document-card.tsx": { "text-success": 1 }, "src/components/documents/document-detail-sheet.tsx": { "toast.success": 2 }, "src/components/documents/documents-view.tsx": { "toast.success": 3 }, + // The document picker's per-row result: tinted only for the `imported` / + // `duplicate` outcome the import route returned for that one document, i.e. + // a document the vault now holds. The run's headline goes through + // WrittenOutcomeLine; a row sits inside a button, where its

cannot. + "src/components/documents/sources/document-source-picker.tsx": { + "text-success": 1, + }, "src/components/error-details.tsx": { CheckCircle2: 2, "text-success": 1 }, "src/components/illness/episode-menu.tsx": { CheckCircle2: 2, diff --git a/src/components/documents/sources/document-source-picker.tsx b/src/components/documents/sources/document-source-picker.tsx index 2199193dc..2c91ce277 100644 --- a/src/components/documents/sources/document-source-picker.tsx +++ b/src/components/documents/sources/document-source-picker.tsx @@ -69,6 +69,9 @@ import { invalidateKeys, queryKeys } from "@/lib/query-keys"; import { cn } from "@/lib/utils"; import type { InboundDocumentKindValue } from "@/lib/validations/inbound-documents"; +import { WrittenOutcomeLine } from "@/components/outcome/written-outcome-line"; +import type { WrittenOutcome } from "@/lib/outcome/written-outcome"; + import { formatBytes } from "../vault-utils"; import { errorCodeOf, @@ -306,7 +309,7 @@ export function DocumentSourcePicker({ data-slot="document-source-selection" > {finished - ? t("documents.sourcePicker.summary", counts) + ? null : running ? t("documents.sourcePicker.importing") : t("documents.sourcePicker.selectedCount", { @@ -469,6 +472,14 @@ export function DocumentSourcePicker({ ) : null} + {finished ? ( + + ) : null} + {stopMessage ? (

0 ? "success" : "empty"; + return held > 0 ? "partial" : "failed"; +} + /** One row's progress or result, under its title. */ function RunLine({ run }: { run: RowRun }) { const { t } = useTranslations(); diff --git a/src/components/settings/integrations/document-sources-card.tsx b/src/components/settings/integrations/document-sources-card.tsx index b198d16ec..d63cfd30d 100644 --- a/src/components/settings/integrations/document-sources-card.tsx +++ b/src/components/settings/integrations/document-sources-card.tsx @@ -16,7 +16,7 @@ * (UI standards §12). */ import { useMutation, useQueryClient } from "@tanstack/react-query"; -import { Archive, CheckCircle2, Loader2, Unlink } from "lucide-react"; +import { Archive, Check, Loader2, Unlink } from "lucide-react"; import { useState } from "react"; import { @@ -38,6 +38,7 @@ import { PasswordInput } from "@/components/ui/password-input"; import { SettingsCardActions } from "@/components/settings/_card-actions"; import { SettingsCardHeader } from "@/components/settings/_card-header"; import { SettingsCard } from "@/components/settings/settings-card"; +import { WrittenOutcomeLine } from "@/components/outcome/written-outcome-line"; import { errorCodeOf, sourceErrorMessage, @@ -111,8 +112,8 @@ export function DocumentSourcesCard() { > {systemName(option)} {isConnected ? ( - ) : null} @@ -328,16 +329,11 @@ function SourceForm({ {outcome ? ( -

- {outcome.message} -

+ ) : null} From 6029c7c30d69dfeeb8a27460f51f466dbb5736bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Sat, 26 Sep 2026 13:45:15 +0200 Subject: [PATCH 09/18] fix(documents): keep a picker row at full contrast once it shows its import result; two date fields per row on phones --- .../documents/sources/document-source-picker.tsx | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/components/documents/sources/document-source-picker.tsx b/src/components/documents/sources/document-source-picker.tsx index 2c91ce277..ba9dca00d 100644 --- a/src/components/documents/sources/document-source-picker.tsx +++ b/src/components/documents/sources/document-source-picker.tsx @@ -415,8 +415,8 @@ export function DocumentSourcePicker({ -
-
+
+
@@ -543,7 +543,9 @@ export function DocumentSourcePicker({ "focus-visible:ring-ring/50 focus-visible:ring-[3px] focus-visible:outline-none", allowed && !running && !finished && "hover:bg-muted/50", on && "border-primary/40 bg-primary/5", - !allowed && "opacity-50", + // A row that just ran keeps full contrast: its result is + // the thing to read. + !allowed && !run && "opacity-50", )} > Date: Sat, 26 Sep 2026 14:20:17 +0200 Subject: [PATCH 10/18] fix(documents): key imports by source instance, keep a token on its origin, and harden the import path - Source keys carry the normalised origin of the Paperless-ngx or Papra they came from (source_instance on the document, the alias and the purge ledger; unique NULLS NOT DISTINCT). A key stored without one (v1.39.2 imports) matches any instance of its system, and a lookup without one matches any stored instance. The upload route, the lookup route and the import script accept or send it; backups carry it. - Saving a connection reuses the stored token only for the same origin; a new origin needs the token again (422 tokenRequired). - Document ids are checked against what each system issues (Paperless digits, Papra [A-Za-z0-9_-], never . or ..) before any URL is built; the Papra organization id likewise. - Paperless API 9 filters dates on created__gte / created__lte, in the picker and the script. A search page the local re-check empties reads on, up to five source pages. Tags are read past the first 250. - The import's key pre-check uses the lookup allowance, a failed fetch no longer charges the upload allowance, and one import per key runs at a time under an advisory lock, so a double click downloads once. - Reading and deleting one's own connection works with the documents module off and the operator list unset. --- .../migration.sql | 24 ++ prisma/schema.prisma | 57 ++-- scripts/__tests__/import-documents.test.ts | 2 + scripts/import-documents.mjs | 9 +- .../documents/inbound/__tests__/route.test.ts | 14 +- src/app/api/documents/inbound/route.ts | 22 +- src/app/api/documents/inbound/source/route.ts | 2 + .../sources/[system]/import/route.ts | 283 ++++++++++++------ .../api/documents/sources/[system]/route.ts | 25 +- .../sources/[system]/search/route.ts | 54 +++- src/app/api/documents/sources/route.ts | 19 +- src/lib/documents/ingest.ts | 22 +- src/lib/documents/source-key.ts | 93 ++++-- .../sources/__tests__/clients.test.ts | 76 ++++- src/lib/documents/sources/clients.ts | 76 +++-- src/lib/documents/sources/route-support.ts | 16 +- src/lib/documents/sources/types.ts | 22 +- src/lib/export/records-backup.ts | 5 + src/lib/export/restore-backup.ts | 9 + src/lib/jobs/document-purge.ts | 7 +- src/lib/validations/backup.ts | 9 + src/lib/validations/document-sources.ts | 5 +- src/lib/validations/inbound-documents.ts | 47 ++- tests/integration/document-sources.test.ts | 197 +++++++++++- 24 files changed, 886 insertions(+), 209 deletions(-) diff --git a/prisma/migrations/0356_document_source_connections/migration.sql b/prisma/migrations/0356_document_source_connections/migration.sql index 6423236ed..f50be788c 100644 --- a/prisma/migrations/0356_document_source_connections/migration.sql +++ b/prisma/migrations/0356_document_source_connections/migration.sql @@ -20,3 +20,27 @@ CREATE UNIQUE INDEX "document_source_connections_user_id_system_key" ALTER TABLE "document_source_connections" ADD CONSTRAINT "document_source_connections_user_id_fkey" FOREIGN KEY ("user_id") REFERENCES "users"("id") ON DELETE CASCADE ON UPDATE CASCADE; + +-- Source instance (#1038 review): the same id in two Paperless-ngx (or two +-- Papra) instances is two documents. A nullable origin joins every source key; +-- keys stored without one (v1.39.2 imports) keep matching any instance, which +-- the application decides at lookup time. NULLS NOT DISTINCT keeps two keys +-- without an instance colliding exactly as before (PG15+; the stack runs 16). +ALTER TABLE "inbound_documents" ADD COLUMN "source_instance" VARCHAR(512); +DROP INDEX IF EXISTS "inbound_documents_user_source_key"; +CREATE UNIQUE INDEX "inbound_documents_user_source_key" + ON "inbound_documents" ("user_id", "source_system", "source_instance", "source_id") + NULLS NOT DISTINCT + WHERE "source_id" IS NOT NULL; + +ALTER TABLE "document_source_aliases" ADD COLUMN "source_instance" VARCHAR(512); +DROP INDEX IF EXISTS "document_source_aliases_user_id_source_system_source_id_key"; +CREATE UNIQUE INDEX "document_source_aliases_user_id_source_system_source_instance_source_id_key" + ON "document_source_aliases" ("user_id", "source_system", "source_instance", "source_id") + NULLS NOT DISTINCT; + +ALTER TABLE "document_import_keys" ADD COLUMN "source_instance" VARCHAR(512); +DROP INDEX IF EXISTS "document_import_keys_user_id_source_system_source_id_key"; +CREATE UNIQUE INDEX "document_import_keys_user_id_source_system_source_instance_source_id_key" + ON "document_import_keys" ("user_id", "source_system", "source_instance", "source_id") + NULLS NOT DISTINCT; diff --git a/prisma/schema.prisma b/prisma/schema.prisma index 47037b890..6c6c06bca 100644 --- a/prisma/schema.prisma +++ b/prisma/schema.prisma @@ -8011,15 +8011,24 @@ model InboundDocument { /// and the Paperless workflow recipe; shown on the detail sheet. sourceSystem String? @map("source_system") @db.VarChar(16) /// v1.39.2 (#1038) — the document's id in `sourceSystem`. Together with - /// `userId` + `sourceSystem` it is unique across live AND tombstoned rows - /// (raw SQL in migration 0353, not expressible in Prisma): - /// CREATE UNIQUE INDEX inbound_documents_user_source_key - /// ON inbound_documents (user_id, source_system, source_id) - /// WHERE source_id IS NOT NULL; + /// `userId`, `sourceSystem` and `sourceInstance` it is unique across live + /// AND tombstoned rows (raw SQL in migrations 0353 and 0356, not expressible + /// in Prisma; see `sourceInstance` for the current definition). /// Including tombstones is the point: re-running an importer must not bring /// back a document the person deleted. The purge carries the key over to /// `DocumentImportKey` when it removes the row. sourceId String? @map("source_id") @db.VarChar(128) + /// v1.39.3 (#1038) — which instance of `sourceSystem` the document came + /// from: the normalised origin (`scheme://host[:port]`) of the Paperless-ngx + /// or Papra it was fetched from, so id 123 on one Paperless and id 123 on + /// another are two keys. Plaintext; an address, not health data. Null for a + /// key sent without one (every v1.39.2 import, an older script, a workflow + /// that does not send it): such a key matches any instance of its system, + /// and a lookup without an instance matches any stored instance. Part of the + /// source-key unique index since migration 0356, which rebuilt it as + /// (user_id, source_system, source_instance, source_id) NULLS NOT DISTINCT + /// WHERE source_id IS NOT NULL. + sourceInstance String? @map("source_instance") @db.VarChar(512) /// v1.39.2 (#1038) — the upload asked for `aiRead=defer`: nothing may send /// this document to an AI provider on its own (the summary catch-up, the /// summary job, lab auto-staging all skip it). Cleared when the person @@ -8081,18 +8090,24 @@ model InboundDocument { /// own, so deleting the document keeps every key that ever reached it deleted. /// Cascades with the document; the purge copies it first. model DocumentSourceAlias { - id String @id @default(cuid()) - userId String @map("user_id") - documentId String @map("document_id") + id String @id @default(cuid()) + userId String @map("user_id") + documentId String @map("document_id") /// Same closed vocabulary as `InboundDocument.sourceSystem`. - sourceSystem String @map("source_system") @db.VarChar(16) - sourceId String @map("source_id") @db.VarChar(128) - createdAt DateTime @default(now()) @map("created_at") + sourceSystem String @map("source_system") @db.VarChar(16) + sourceId String @map("source_id") @db.VarChar(128) + /// v1.39.3 — the source instance, as on `InboundDocument.sourceInstance` + /// (null matches any instance of the system). + sourceInstance String? @map("source_instance") @db.VarChar(512) + createdAt DateTime @default(now()) @map("created_at") user User @relation(fields: [userId], references: [id], onDelete: Cascade) document InboundDocument @relation(fields: [documentId], references: [id], onDelete: Cascade) - @@unique([userId, sourceSystem, sourceId]) + // NULLS NOT DISTINCT in migration 0356 (hand-written; Prisma would + // regenerate it NULLS DISTINCT): a key stored without an instance still + // collides with the same key without one. + @@unique([userId, sourceSystem, sourceInstance, sourceId]) @@index([documentId]) @@map("document_source_aliases") } @@ -8104,16 +8119,22 @@ model DocumentSourceAlias { /// "deleted" instead of storing the document again. Written by the purge only; /// read by the upload route's source-key check. Holds no document content. model DocumentImportKey { - id String @id @default(cuid()) - userId String @map("user_id") + id String @id @default(cuid()) + userId String @map("user_id") /// Same closed vocabulary as `InboundDocument.sourceSystem`. - sourceSystem String @map("source_system") @db.VarChar(16) - sourceId String @map("source_id") @db.VarChar(128) - createdAt DateTime @default(now()) @map("created_at") + sourceSystem String @map("source_system") @db.VarChar(16) + sourceId String @map("source_id") @db.VarChar(128) + /// v1.39.3 — the source instance, as on `InboundDocument.sourceInstance` + /// (null matches any instance of the system). + sourceInstance String? @map("source_instance") @db.VarChar(512) + createdAt DateTime @default(now()) @map("created_at") user User @relation(fields: [userId], references: [id], onDelete: Cascade) - @@unique([userId, sourceSystem, sourceId]) + // NULLS NOT DISTINCT in migration 0356 (hand-written; Prisma would + // regenerate it NULLS DISTINCT): a key stored without an instance still + // collides with the same key without one. + @@unique([userId, sourceSystem, sourceInstance, sourceId]) @@map("document_import_keys") } diff --git a/scripts/__tests__/import-documents.test.ts b/scripts/__tests__/import-documents.test.ts index 6c5865fa4..994ded97e 100644 --- a/scripts/__tests__/import-documents.test.ts +++ b/scripts/__tests__/import-documents.test.ts @@ -424,6 +424,7 @@ describe("import-documents.mjs — Paperless-ngx", () => { kind: "LAB_RESULT", sourceSystem: "PAPERLESS", sourceId: "1", + sourceInstance: new URL(pl.url).origin, aiRead: "defer", }, }); @@ -664,6 +665,7 @@ describe("import-documents.mjs — Papra", () => { documentDate: "2019-06-01", sourceSystem: "PAPRA", sourceId: "doc_a", + sourceInstance: new URL(pp.url).origin, }); // --ai-read: no deferral field at all. expect(sent[0].aiRead).toBeUndefined(); diff --git a/scripts/import-documents.mjs b/scripts/import-documents.mjs index 1394136b3..c909a35f2 100644 --- a/scripts/import-documents.mjs +++ b/scripts/import-documents.mjs @@ -301,6 +301,7 @@ function paperless(options) { return { system: "PAPERLESS", + instance: new URL(base).origin, label, async *documents() { const query = new URLSearchParams({ @@ -324,7 +325,8 @@ function paperless(options) { tagId = found[0].id; query.set("tags__id__all", String(tagId)); } - if (options.since) query.set("created__date__gte", options.since); + // API version 9 made `created` a plain date: filter on it directly. + if (options.since) query.set("created__gte", options.since); let typeNames = new Map(); if (options.kindMap.size > 0) { @@ -399,6 +401,7 @@ function papra(options) { return { system: "PAPRA", + instance: new URL(options.baseUrl).origin, label, async *documents() { let tagId = null; @@ -526,6 +529,7 @@ function keyQuery(source, doc) { return new URLSearchParams({ sourceSystem: source.system, sourceId: doc.sourceId.slice(0, SOURCE_ID_MAX), + sourceInstance: source.instance, }); } @@ -617,6 +621,9 @@ async function upload(options, source, doc, file) { if (!options.aiRead) form.append("aiRead", "defer"); form.append("sourceSystem", source.system); form.append("sourceId", doc.sourceId.slice(0, SOURCE_ID_MAX)); + // Which Paperless-ngx or Papra this id belongs to: the same id in two + // instances is two documents. HealthLog 1.39.2 ignores the field. + form.append("sourceInstance", source.instance); return form; }, ); diff --git a/src/app/api/documents/inbound/__tests__/route.test.ts b/src/app/api/documents/inbound/__tests__/route.test.ts index bcb479701..a83d87578 100644 --- a/src/app/api/documents/inbound/__tests__/route.test.ts +++ b/src/app/api/documents/inbound/__tests__/route.test.ts @@ -51,10 +51,10 @@ vi.mock("@/lib/db", () => { findFirst: vi.fn(), }, documentImportKey: { - findUnique: vi.fn(), + findFirst: vi.fn(), }, documentSourceAlias: { - findUnique: vi.fn(), + findFirst: vi.fn(), createMany: vi.fn(), count: vi.fn(), }, @@ -239,10 +239,10 @@ beforeEach(() => { [] as never, ); vi.mocked(prisma.inboundDocument.findFirst).mockResolvedValue(null as never); - vi.mocked(prisma.documentImportKey.findUnique).mockResolvedValue( + vi.mocked(prisma.documentImportKey.findFirst).mockResolvedValue( null as never, ); - vi.mocked(prisma.documentSourceAlias.findUnique).mockResolvedValue( + vi.mocked(prisma.documentSourceAlias.findFirst).mockResolvedValue( null as never, ); vi.mocked(prisma.documentSourceAlias.createMany).mockResolvedValue({ @@ -783,7 +783,7 @@ describe("POST /api/documents/inbound — document import", () => { }); it("remembers a purged import through the ledger", async () => { - vi.mocked(prisma.documentImportKey.findUnique).mockResolvedValue({ + vi.mocked(prisma.documentImportKey.findFirst).mockResolvedValue({ id: "key-1", } as never); const res = await post( @@ -909,6 +909,8 @@ describe("POST /api/documents/inbound — document import", () => { userId: "user-1", documentId: "doc-manual", sourceSystem: "PAPRA", + // Sent without an instance: the alias matches any Papra. + sourceInstance: null, sourceId: "doc_9", }, ], @@ -936,7 +938,7 @@ describe("POST /api/documents/inbound — document import", () => { }); it("resolves a key held as an alias of a deleted document to deleted", async () => { - vi.mocked(prisma.documentSourceAlias.findUnique).mockResolvedValue({ + vi.mocked(prisma.documentSourceAlias.findFirst).mockResolvedValue({ document: docRow({ id: "doc-gone", deletedAt: new Date() }), } as never); const res = await post( diff --git a/src/app/api/documents/inbound/route.ts b/src/app/api/documents/inbound/route.ts index 90c4dde70..f2c6723d4 100644 --- a/src/app/api/documents/inbound/route.ts +++ b/src/app/api/documents/inbound/route.ts @@ -274,14 +274,17 @@ async function processUpload( const url = new URL(request.url); const querySystem = url.searchParams.get("sourceSystem"); const queryId = url.searchParams.get("sourceId"); + const queryInstance = url.searchParams.get("sourceInstance"); let queryKey: { sourceSystem: DocumentSourceSystemValue; sourceId: string; + sourceInstance?: string; } | null = null; if (querySystem !== null || queryId !== null) { const parsedKey = documentSourceKeySchema.safeParse({ sourceSystem: querySystem ?? undefined, sourceId: queryId ?? undefined, + sourceInstance: queryInstance ?? undefined, }); if (!parsedKey.success) { return apiValidationError( @@ -312,6 +315,7 @@ async function processUpload( user.id, queryKey.sourceSystem, queryKey.sourceId, + queryKey.sourceInstance ?? null, ); if (match) { return answerSourceKey( @@ -408,6 +412,7 @@ async function processUpload( encounterIds: rawEncounterIds.length > 0 ? rawEncounterIds : undefined, sourceSystem: formData.get("sourceSystem") ?? undefined, sourceId: formData.get("sourceId") ?? undefined, + sourceInstance: formData.get("sourceInstance") ?? undefined, aiRead: formData.get("aiRead") ?? undefined, }); if (!parsed.success) { @@ -425,10 +430,14 @@ async function processUpload( // both they have to agree. const formSystem = parsed.data.sourceSystem ?? null; const formId = formSystem ? (parsed.data.sourceId ?? null) : null; + const formInstance = formId ? (parsed.data.sourceInstance ?? null) : null; if ( queryKey && formSystem !== null && - (formSystem !== queryKey.sourceSystem || formId !== queryKey.sourceId) + (formSystem !== queryKey.sourceSystem || + formId !== queryKey.sourceId || + (formInstance !== null && + formInstance !== (queryKey.sourceInstance ?? null))) ) { return apiError( "The source key in the address and in the form differ.", @@ -438,6 +447,9 @@ async function processUpload( } const sourceSystem = queryKey?.sourceSystem ?? formSystem; const sourceId = queryKey?.sourceId ?? formId; + const sourceInstance = sourceId + ? (queryKey?.sourceInstance ?? formInstance ?? null) + : null; const ctx: UploadContext = { userId: user.id, scoped, @@ -451,7 +463,12 @@ async function processUpload( // count (the unique index has no `deleted_at` predicate), and past the purge // the ledger remembers. if (sourceSystem && sourceId && !queryKey) { - const match = await findSourceKey(user.id, sourceSystem, sourceId); + const match = await findSourceKey( + user.id, + sourceSystem, + sourceId, + sourceInstance, + ); if (match) return answerSourceKey(ctx, match); } @@ -478,6 +495,7 @@ async function processUpload( encounterIds: parsed.data.encounterIds ?? [], sourceSystem, sourceId, + sourceInstance, aiDeferred: ctx.aiDeferred, sourceKeyChecked: true, limits, diff --git a/src/app/api/documents/inbound/source/route.ts b/src/app/api/documents/inbound/source/route.ts index caada8b66..ec675a97c 100644 --- a/src/app/api/documents/inbound/source/route.ts +++ b/src/app/api/documents/inbound/source/route.ts @@ -57,6 +57,7 @@ export const GET = apiHandler(async (request: Request) => { const parsed = documentSourceKeySchema.safeParse({ sourceSystem: url.searchParams.get("sourceSystem") ?? undefined, sourceId: url.searchParams.get("sourceId") ?? undefined, + sourceInstance: url.searchParams.get("sourceInstance") ?? undefined, }); if (!parsed.success) { return apiValidationError( @@ -71,6 +72,7 @@ export const GET = apiHandler(async (request: Request) => { user.id, parsed.data.sourceSystem, parsed.data.sourceId, + parsed.data.sourceInstance ?? null, ); annotate({ action: { name: "documents.vault.sourceLookup" }, diff --git a/src/app/api/documents/sources/[system]/import/route.ts b/src/app/api/documents/sources/[system]/import/route.ts index 606e3c947..7da5933af 100644 --- a/src/app/api/documents/sources/[system]/import/route.ts +++ b/src/app/api/documents/sources/[system]/import/route.ts @@ -12,13 +12,16 @@ * imported (or already holding this key as an alias) is a `duplicate`, * and one the person deleted here is `deleted` — nothing is fetched and * nothing is stored, exactly as the token upload answers the same key; - * 3. only then the vault's own upload allowance (60 an hour, shared with - * uploads from the web and the phone) and an upload slot are taken, the - * document's details and its original file are fetched (bounded by the - * vault's per-file cap), and the file goes through `ingestDocument`, the - * same function the upload route stores through: type check, content - * dedup, quota, provenance (`sourceSystem` / `sourceId`), the background - * jobs; + * 3. only then is the source asked (on the search allowance, like every + * request to it) and an upload slot taken. One import per key and person + * runs at a time under an advisory lock, so a double click downloads the + * document once. The details and the original file are fetched (bounded + * by the vault's per-file cap); only a file that arrived is charged to + * the vault's own upload allowance (60 an hour, shared with uploads from + * the web and the phone), and it goes through `ingestDocument`, the same + * function the upload route stores through: type check, content dedup, + * quota, provenance (`sourceSystem` / `sourceId` / `sourceInstance`), the + * background jobs; * 4. the document is linked to the record through `src/lib/links/`. * * AI reading follows the person's own setting, unlike the token upload's @@ -48,18 +51,26 @@ import { UPLOAD_LIMIT_PER_HOUR, UPLOAD_WINDOW_MS, } from "@/lib/documents/ingest"; -import { findSourceKey, type SourceKeyMatch } from "@/lib/documents/source-key"; +import { + checkSourceLookupRateLimit, + findSourceKey, + type SourceKeyMatch, +} from "@/lib/documents/source-key"; +import type { SourceClient } from "@/lib/documents/sources/clients"; import { clientFor, loadConnection } from "@/lib/documents/sources/connections"; import { admitDocumentSourceCaller, + checkSearchRateLimit, responseForSourceFailure, sourceErrorResponse, systemParam, } from "@/lib/documents/sources/route-support"; -import type { - DocumentImportOutcome, - DocumentPickerLinkKind, - DocumentSourceImportDto, +import { + isSourceDocumentId, + type DocumentImportOutcome, + type DocumentPickerLinkKind, + type DocumentPickerSystem, + type DocumentSourceImportDto, } from "@/lib/documents/sources/types"; import { acquireDocumentUploadSlot, @@ -69,11 +80,17 @@ import { linkTargets } from "@/lib/links"; import { annotate } from "@/lib/logging/context"; import { checkRateLimit, rateLimitHeaders } from "@/lib/rate-limit"; import { documentSourceImportSchema } from "@/lib/validations/document-sources"; +import type { InboundDocumentKindValue } from "@/lib/validations/inbound-documents"; export const dynamic = "force-dynamic"; type RouteParams = { params: Promise<{ system: string }> }; +/** How long an import waits for a database connection to hold its lock on. */ +const IMPORT_LOCK_WAIT_MS = 10_000; +/** Longest a single import may hold its key: the download plus the store. */ +const IMPORT_LOCK_HOLD_MS = 150_000; + interface LinkTarget { kind: DocumentPickerLinkKind; id: string; @@ -172,6 +189,10 @@ export const POST = apiHandler( ); } const { sourceId, kind, link } = parsed.data; + // Only an id this system could have issued goes into a request path. + if (!isSourceDocumentId(system, sourceId)) { + return apiError("Not a document id of this archive.", 422); + } const row = await loadConnection(userId, system); if (!row) return sourceErrorResponse("notConnected"); @@ -181,25 +202,38 @@ export const POST = apiHandler( } catch (err) { return responseForSourceFailure(err); } + const instance = client.instance; if (link && !(await ownsLinkTarget(userId, link))) { return sourceErrorResponse("linkTargetNotFound"); } - // The source key answers before anything is downloaded or charged. - const held = await findSourceKey(userId, system, sourceId); + // The source key answers before anything is downloaded or charged, on the + // same metered lookup allowance the upload's key check draws on. + const lookupRl = await checkSourceLookupRateLimit( + false, + auth.session.id, + userId, + ); + if (!lookupRl.allowed) { + return sourceErrorResponse( + "rateLimited", + undefined, + rateLimitHeaders(lookupRl), + ); + } + const held = await findSourceKey(userId, system, sourceId, instance); if (held) return answerHeldKey(system, userId, link, held); - const rl = await checkRateLimit( - personalUploadBucket(userId), - UPLOAD_LIMIT_PER_HOUR, - UPLOAD_WINDOW_MS, - ); - if (!rl.allowed) { - return apiError("Too many uploads. Try again later.", 429, { - errorCode: "documents.inbound.rateLimited", - headers: rateLimitHeaders(rl), - }); + // A request to the source counts against the search allowance like any + // other; the upload allowance is charged only once a file arrived. + const fetchRl = await checkSearchRateLimit(userId); + if (!fetchRl.allowed) { + return sourceErrorResponse( + "rateLimited", + undefined, + rateLimitHeaders(fetchRl), + ); } const release = acquireDocumentUploadSlot(userId); @@ -216,83 +250,136 @@ export const POST = apiHandler( } try { - const limits = await resolveDocumentLimits(userId); - let meta; - let file; - try { - meta = await client.document(sourceId); - file = await client.download(sourceId, limits.maxFileBytes); - } catch (err) { - return responseForSourceFailure(err); - } - - const result = await ingestDocument({ - userId, - scoped: false, - ipAddress: getClientIp(request), - bytes: Buffer.from(file.bytes), - filename: meta.filename ?? file.filename, - title: meta.title, - kind: kind ?? null, - documentDate: meta.date, - episodeIds: [], - encounterIds: [], - sourceSystem: system, - sourceId, - aiDeferred: false, - sourceKeyChecked: true, - limits, - // The import is data; only the summary queued after a fresh insert - // follows the capability (and the person's auto-read setting). - documentAi: () => getAiCapability("documentAi"), - }); - - switch (result.kind) { - case "sourceKey": - return answerHeldKey(system, userId, link, result.match); - case "duplicate": { - // The same bytes were already in the vault (an upload, or another - // archive); the key is now remembered for that document. - const linked = await linkDocument(userId, link, result.document.id); - return respond(system, "duplicate", result.document.id, linked); - } - case "stored": { - const linked = await linkDocument(userId, link, result.document.id); - return respond(system, "imported", result.document.id, linked); - } - case "tooLarge": - return apiError("File is too large.", 413, { - errorCode: "documents.inbound.fileTooLarge", - reason: "fileTooLarge", - maxFileBytes: result.maxFileBytes, + // One import of one key at a time per person: a second tab or a double + // click waits here, then finds the key the first one stored, so the + // document is downloaded once. The lock is transaction-scoped and the + // transaction holds nothing else; the store runs in its own. + return await prisma.$transaction( + async (tx) => { + await tx.$queryRaw` + SELECT 1 AS locked + FROM pg_advisory_xact_lock(hashtextextended(${`documents-import:${userId}:${system}:${instance}:${sourceId}`}, 0)) + `; + const raced = await findSourceKey(userId, system, sourceId, instance); + if (raced) return answerHeldKey(system, userId, link, raced); + return importOne({ + request, + client, + userId, + system, + instance, + sourceId, + kind: kind ?? null, + link, }); - case "empty": - case "unsupportedType": - return apiError("This file type is not supported.", 415, { - errorCode: "documents.inbound.fileType", - reason: "unsupportedType", - }); - case "quotaExceeded": - return apiError("Storage quota exceeded.", 413, { - errorCode: "documents.inbound.quotaExceeded", - reason: "quotaExceeded", - quotaBytes: result.quotaBytes, - usedBytes: result.usedBytes, - }); - case "aliasLimit": - return apiError( - "This file is already stored under too many source ids.", - 409, - { errorCode: "documents.inbound.sourceAliasLimit" }, - ); - case "episodeNotFound": - case "encounterNotFound": - // No pre-links are passed on this path; unreachable, but a result - // the switch does not name must not fall through silently. - return sourceErrorResponse("linkTargetNotFound"); - } + }, + { maxWait: IMPORT_LOCK_WAIT_MS, timeout: IMPORT_LOCK_HOLD_MS }, + ); } finally { release(); } }, ); + +/** Fetch one document and store it; the caller holds the key's lock. */ +async function importOne(args: { + request: Request; + client: SourceClient; + userId: string; + system: DocumentPickerSystem; + instance: string; + sourceId: string; + kind: InboundDocumentKindValue | null; + link: LinkTarget | undefined; +}): Promise { + const { request, client, userId, system, instance, sourceId, kind, link } = + args; + const limits = await resolveDocumentLimits(userId); + let meta; + let file; + try { + meta = await client.document(sourceId); + file = await client.download(sourceId, limits.maxFileBytes); + } catch (err) { + return responseForSourceFailure(err); + } + + const rl = await checkRateLimit( + personalUploadBucket(userId), + UPLOAD_LIMIT_PER_HOUR, + UPLOAD_WINDOW_MS, + ); + if (!rl.allowed) { + return apiError("Too many uploads. Try again later.", 429, { + errorCode: "documents.inbound.rateLimited", + headers: rateLimitHeaders(rl), + }); + } + + const result = await ingestDocument({ + userId, + scoped: false, + ipAddress: getClientIp(request), + bytes: Buffer.from(file.bytes), + filename: meta.filename ?? file.filename, + title: meta.title, + kind, + documentDate: meta.date, + episodeIds: [], + encounterIds: [], + sourceSystem: system, + sourceId, + sourceInstance: instance, + aiDeferred: false, + sourceKeyChecked: true, + limits, + // The import is data; only the summary queued after a fresh insert + // follows the capability (and the person's auto-read setting). + documentAi: () => getAiCapability("documentAi"), + }); + + switch (result.kind) { + case "sourceKey": + return answerHeldKey(system, userId, link, result.match); + case "duplicate": { + // The same bytes were already in the vault (an upload, or another + // archive); the key is now remembered for that document. + const linked = await linkDocument(userId, link, result.document.id); + return respond(system, "duplicate", result.document.id, linked); + } + case "stored": { + const linked = await linkDocument(userId, link, result.document.id); + return respond(system, "imported", result.document.id, linked); + } + case "tooLarge": + return apiError("File is too large.", 413, { + errorCode: "documents.inbound.fileTooLarge", + reason: "fileTooLarge", + maxFileBytes: result.maxFileBytes, + }); + case "empty": + case "unsupportedType": + return apiError("This file type is not supported.", 415, { + errorCode: "documents.inbound.fileType", + reason: "unsupportedType", + }); + case "quotaExceeded": + return apiError("Storage quota exceeded.", 413, { + errorCode: "documents.inbound.quotaExceeded", + reason: "quotaExceeded", + quotaBytes: result.quotaBytes, + usedBytes: result.usedBytes, + }); + case "aliasLimit": + return apiError( + "This file is already stored under too many source ids.", + 409, + { errorCode: "documents.inbound.sourceAliasLimit" }, + ); + case "episodeNotFound": + case "encounterNotFound": + // No pre-links are passed on this path; unreachable, but a result + // the switch does not name must not fall through silently. + return sourceErrorResponse("linkTargetNotFound"); + } +} diff --git a/src/app/api/documents/sources/[system]/route.ts b/src/app/api/documents/sources/[system]/route.ts index 0057e4c24..91cfb0dba 100644 --- a/src/app/api/documents/sources/[system]/route.ts +++ b/src/app/api/documents/sources/[system]/route.ts @@ -6,8 +6,9 @@ * (`DOCUMENT_SOURCE_ORIGINS`), tests the connection live before storing * anything (a wrong token or an unreachable host is reported here, not on the * first search), then stores the token encrypted. A PUT without a token keeps - * the stored one, so the address or the Papra organization can change without - * typing the key again; the first save must carry it. The response is the + * the stored one, so the base path or the Papra organization can change + * without typing the key again, but only on the same origin: a new origin, and + * the first save, must carry it (422 `documents.sources.tokenRequired`). The response is the * connection without its token, like every read of it. * * DELETE removes the row, and with it the token. @@ -100,8 +101,16 @@ export const PUT = apiHandler( } const existing = await loadConnection(userId, system); + // The stored token is only ever sent back to the origin it was saved + // for. A new address on another origin needs the token typed again, or + // saving an address would hand the old archive's token to whatever the + // new one is. + const reusable = + existing !== null && + evaluateSourceBaseUrl(existing.baseUrl, new Set([verdict.origin])).ok; const token = - parsed.data.token ?? (existing ? decrypt(existing.tokenEncrypted) : null); + parsed.data.token ?? + (existing && reusable ? decrypt(existing.tokenEncrypted) : null); if (!token) { return apiError("Enter the API token.", 422, { errorCode: "documents.sources.tokenRequired", @@ -165,9 +174,13 @@ export const PUT = apiHandler( export const DELETE = apiHandler( async (request: NextRequest, { params }: RouteParams) => { const auth = await requireAuth(); - // No list needed to forget a connection: an operator who removed the - // origin must not leave the person unable to delete the stored token. - const refused = await admitDocumentSourceCaller(auth, { needsList: false }); + // Neither the list nor the module is needed to forget a connection: an + // operator who removed the origin, or a person who switched Documents + // off, must still be able to delete the stored token. + const refused = await admitDocumentSourceCaller(auth, { + needsList: false, + needsModule: false, + }); if (refused) return refused; const system = await systemParam(params); if (system instanceof Response) return system; diff --git a/src/app/api/documents/sources/[system]/search/route.ts b/src/app/api/documents/sources/[system]/search/route.ts index 7296048e8..edaf7a9c3 100644 --- a/src/app/api/documents/sources/[system]/search/route.ts +++ b/src/app/api/documents/sources/[system]/search/route.ts @@ -9,6 +9,10 @@ * `new`, `imported` (with the vault document's id) or `deleted` (the person * deleted it here; importing it again is refused). Nothing is stored. * + * When the source ignores a filter and the local re-check empties a page, the + * route reads on (up to five source pages) and answers with the last page it + * read, so the client's "load more" continues after it. + * * Sixty searches a minute, shared with the tag read. Cookie-only and * owner-only; see `admitDocumentSourceCaller`. */ @@ -30,9 +34,10 @@ import { sourceErrorResponse, systemParam, } from "@/lib/documents/sources/route-support"; -import type { - DocumentSourceResultDto, - DocumentSourceSearchDto, +import { + DOCUMENT_PICKER_MAX_PAGE, + type DocumentSourceResultDto, + type DocumentSourceSearchDto, } from "@/lib/documents/sources/types"; import { annotate } from "@/lib/logging/context"; import { rateLimitHeaders } from "@/lib/rate-limit"; @@ -40,6 +45,9 @@ import { documentSourceSearchSchema } from "@/lib/validations/document-sources"; export const dynamic = "force-dynamic"; +/** Source pages one search request may read to find a non-empty one. */ +const SEARCH_PAGES_PER_REQUEST = 5; + type RouteParams = { params: Promise<{ system: string }> }; export const GET = apiHandler( @@ -77,16 +85,34 @@ export const GET = apiHandler( ); } - let found: { items: SourceListItem[]; hasMore: boolean }; + // A page the local tag/date re-check emptied is not an answer: the next + // source page may hold matches. Read on, a bounded number of pages, and + // report the last one read so "load more" continues after it. + let found: { items: SourceListItem[]; hasMore: boolean } = { + items: [], + hasMore: false, + }; + let lastPage = page; + let instance: string; try { const client = await openConnection(userId, system); - found = await client.search({ - q, - tagId: tag ?? null, - from: from ?? null, - to: to ?? null, - page, - }); + instance = client.instance; + for ( + let next = page; + next < page + SEARCH_PAGES_PER_REQUEST && + next <= DOCUMENT_PICKER_MAX_PAGE; + next += 1 + ) { + found = await client.search({ + q, + tagId: tag ?? null, + from: from ?? null, + to: to ?? null, + page: next, + }); + lastPage = next; + if (found.items.length > 0 || !found.hasMore) break; + } } catch (err) { return responseForSourceFailure(err); } @@ -95,6 +121,7 @@ export const GET = apiHandler( userId, system, found.items.map((item) => item.sourceId), + instance, ); const results: DocumentSourceResultDto[] = found.items.map((item) => { const held = states.get(item.sourceId); @@ -116,6 +143,7 @@ export const GET = apiHandler( meta: { system, page, + pagesRead: lastPage - page + 1, count: results.length, filtered: Boolean(q || tag || from || to), }, @@ -123,8 +151,8 @@ export const GET = apiHandler( const body: DocumentSourceSearchDto = { results, - page, - hasMore: found.hasMore, + page: lastPage, + hasMore: found.hasMore && lastPage < DOCUMENT_PICKER_MAX_PAGE, }; return apiSuccess(body); }, diff --git a/src/app/api/documents/sources/route.ts b/src/app/api/documents/sources/route.ts index 0a91b184b..e79f4b22f 100644 --- a/src/app/api/documents/sources/route.ts +++ b/src/app/api/documents/sources/route.ts @@ -8,9 +8,10 @@ * No request leaves the server here. * * Cookie-only and owner-only like every route under this path - * (`admitDocumentSourceCaller`). With the list unset it answers 200 - * `available: false` rather than 404, because this is the one read the client - * makes to decide whether to show the picker at all. + * (`admitDocumentSourceCaller`), but it needs neither the documents module nor + * the list: with the list unset it answers 200 `available: false` (plus any + * rows still stored, so they can be deleted) rather than 404, because this is + * the one read the client makes to decide what to show. */ import { apiHandler, requireAuth } from "@/lib/api-handler"; import { apiSuccess } from "@/lib/api-response"; @@ -30,21 +31,27 @@ export const dynamic = "force-dynamic"; export const GET = apiHandler(async () => { const auth = await requireAuth(); - const refused = await admitDocumentSourceCaller(auth, { needsList: false }); + const refused = await admitDocumentSourceCaller(auth, { + needsList: false, + needsModule: false, + }); if (refused) return refused; annotate({ action: { name: "documents.sources.status" } }); + // Rows are listed even with the picker off: a connection whose origin the + // operator removed (or whose list is gone) must stay visible so it can be + // deleted. `originAllowed` is false for each of them then. + const rows = await listConnections(auth.user.id); if (!documentSourcesAvailable()) { const off: DocumentSourcesStatusDto = { available: false, allowedOrigins: [], - connections: [], + connections: rows.map(toConnectionDto), }; return apiSuccess(off); } - const rows = await listConnections(auth.user.id); const status: DocumentSourcesStatusDto = { available: true, allowedOrigins: listedDocumentSourceOrigins(), diff --git a/src/lib/documents/ingest.ts b/src/lib/documents/ingest.ts index e38df19d3..2e21291c4 100644 --- a/src/lib/documents/ingest.ts +++ b/src/lib/documents/ingest.ts @@ -93,6 +93,11 @@ export interface IngestInput { encounterIds: string[]; sourceSystem: DocumentSourceSystemValue | null; sourceId: string | null; + /** + * The source instance (origin) of the key, or null for a key sent without + * one; see `src/lib/documents/source-key.ts` for how null matches. + */ + sourceInstance: string | null; /** Hold back automatic AI reading (`aiRead=defer`). */ aiDeferred: boolean; /** @@ -151,6 +156,7 @@ async function answerDuplicate( existing.id, input.sourceSystem, input.sourceId, + input.sourceInstance, ); if (remembered === "limit") return { kind: "aliasLimit" }; } @@ -161,6 +167,7 @@ export async function ingestDocument( input: IngestInput, ): Promise { const { userId, sourceSystem, sourceId, limits, bytes } = input; + const sourceInstance = sourceId ? input.sourceInstance : null; // A source key answers before the bytes are looked at: an import re-sending // what it sent before is a duplicate, and one re-sending a document the @@ -168,7 +175,12 @@ export async function ingestDocument( // count (the unique index has no `deleted_at` predicate), and past the purge // the ledger remembers. if (sourceSystem && sourceId && !input.sourceKeyChecked) { - const match = await findSourceKey(userId, sourceSystem, sourceId); + const match = await findSourceKey( + userId, + sourceSystem, + sourceId, + sourceInstance, + ); if (match) return { kind: "sourceKey", match }; } @@ -258,6 +270,7 @@ export async function ingestDocument( : new Date(), sourceSystem, sourceId, + sourceInstance, aiReadDeferred: input.aiDeferred, }, omit: { contentEncrypted: true }, @@ -296,7 +309,12 @@ export async function ingestDocument( // source key, or the same bytes. Surface the winner exactly as the fast // paths above would have. if (sourceSystem && sourceId) { - const match = await findSourceKey(userId, sourceSystem, sourceId); + const match = await findSourceKey( + userId, + sourceSystem, + sourceId, + sourceInstance, + ); if (match) return { kind: "sourceKey", match }; } const winner = await prisma.inboundDocument.findFirst({ diff --git a/src/lib/documents/source-key.ts b/src/lib/documents/source-key.ts index 61fa6580d..f089aa3b4 100644 --- a/src/lib/documents/source-key.ts +++ b/src/lib/documents/source-key.ts @@ -11,8 +11,21 @@ * 3. `DocumentImportKey` — a key whose document the purge has removed. * * The first two resolve to a document (live → duplicate, tombstoned → - * deleted); the third only to "deleted". The upload route and the lookup - * route both ask here, so they cannot disagree about what a key means. + * deleted); the third only to "deleted". The upload route, the lookup route + * and the document picker all ask here, so they cannot disagree about what a + * key means. + * + * v1.39.3 — a key also names its instance (`sourceInstance`, the origin of + * the Paperless-ngx or Papra it came from): id 123 on one Paperless and id + * 123 on another are different documents. A null on either side is a + * wildcard, so nothing stored before the column existed is forgotten: + * + * - a key stored without an instance (a v1.39.2 import, an older script, a + * workflow that does not send one) matches a lookup for any instance of + * its system; + * - a lookup without an instance matches a key stored for any instance. + * + * Where both an exact and a wildcard match exist, the exact one wins. */ import { prisma } from "@/lib/db"; import { checkRateLimit, type RateLimitResult } from "@/lib/rate-limit"; @@ -23,22 +36,44 @@ export type SourceKeyMatch = | { state: "live"; document: SerialisableDocument } | { state: "deleted"; id: string | null }; +/** + * The instance half of a key match: the same instance or a stored key without + * one; any instance when the lookup has none. + */ +export function instanceMatch(sourceInstance: string | null): { + OR?: Array<{ sourceInstance: string | null }>; +} { + return sourceInstance === null + ? {} + : { OR: [{ sourceInstance }, { sourceInstance: null }] }; +} + +/** Exact instance first, then a key stored without one (nulls sort last). */ +const EXACT_FIRST = { sourceInstance: { sort: "asc", nulls: "last" } } as const; + export async function findSourceKey( userId: string, sourceSystem: DocumentSourceSystemValue, sourceId: string, + sourceInstance: string | null = null, ): Promise { + const key = { + userId, + sourceSystem, + sourceId, + ...instanceMatch(sourceInstance), + }; const own = await prisma.inboundDocument.findFirst({ - where: { userId, sourceSystem, sourceId }, + where: key, omit: { contentEncrypted: true }, + orderBy: EXACT_FIRST, }); const viaAlias = own ? null - : await prisma.documentSourceAlias.findUnique({ - where: { - userId_sourceSystem_sourceId: { userId, sourceSystem, sourceId }, - }, + : await prisma.documentSourceAlias.findFirst({ + where: key, select: { document: { omit: { contentEncrypted: true } } }, + orderBy: EXACT_FIRST, }); const document = own ?? viaAlias?.document ?? null; if (document) { @@ -46,10 +81,8 @@ export async function findSourceKey( ? { state: "deleted", id: document.id } : { state: "live", document }; } - const purged = await prisma.documentImportKey.findUnique({ - where: { - userId_sourceSystem_sourceId: { userId, sourceSystem, sourceId }, - }, + const purged = await prisma.documentImportKey.findFirst({ + where: key, select: { id: true }, }); return purged ? { state: "deleted", id: null } : null; @@ -74,13 +107,14 @@ export async function rememberSourceAlias( documentId: string, sourceSystem: DocumentSourceSystemValue, sourceId: string, + sourceInstance: string | null = null, ): Promise<"remembered" | "known" | "limit"> { const held = await prisma.documentSourceAlias.count({ where: { userId, documentId }, }); if (held >= MAX_SOURCE_ALIASES_PER_DOCUMENT) return "limit"; const { count } = await prisma.documentSourceAlias.createMany({ - data: [{ userId, documentId, sourceSystem, sourceId }], + data: [{ userId, documentId, sourceSystem, sourceInstance, sourceId }], skipDuplicates: true, }); return count > 0 ? "remembered" : "known"; @@ -123,33 +157,56 @@ export async function findSourceKeyStates( userId: string, sourceSystem: DocumentSourceSystemValue, sourceIds: string[], + sourceInstance: string | null = null, ): Promise> { const states = new Map(); const ids = [...new Set(sourceIds)]; if (ids.length === 0) return states; + const where = { + userId, + sourceSystem, + sourceId: { in: ids }, + ...instanceMatch(sourceInstance), + }; const [own, aliases, purged] = await Promise.all([ prisma.inboundDocument.findMany({ - where: { userId, sourceSystem, sourceId: { in: ids } }, - select: { id: true, sourceId: true, deletedAt: true }, + where, + select: { + id: true, + sourceId: true, + sourceInstance: true, + deletedAt: true, + }, }), prisma.documentSourceAlias.findMany({ - where: { userId, sourceSystem, sourceId: { in: ids } }, + where, select: { sourceId: true, + sourceInstance: true, document: { select: { id: true, deletedAt: true } }, }, }), prisma.documentImportKey.findMany({ - where: { userId, sourceSystem, sourceId: { in: ids } }, + where, select: { sourceId: true }, }), ]); + // A wildcard (instance-less) row first, so an exact-instance row after it + // wins, as it does in `findSourceKey`. + const wildcardFirst = ( + rows: T[], + ) => + [...rows].sort( + (a, b) => + Number(a.sourceInstance === sourceInstance) - + Number(b.sourceInstance === sourceInstance), + ); // Lowest precedence first, so the document's own key wins, as it does in // `findSourceKey`. for (const row of purged) states.set(row.sourceId, { state: "deleted" }); - for (const row of aliases) { + for (const row of wildcardFirst(aliases)) { states.set( row.sourceId, row.document.deletedAt @@ -157,7 +214,7 @@ export async function findSourceKeyStates( : { state: "imported", documentId: row.document.id }, ); } - for (const row of own) { + for (const row of wildcardFirst(own)) { if (!row.sourceId) continue; states.set( row.sourceId, diff --git a/src/lib/documents/sources/__tests__/clients.test.ts b/src/lib/documents/sources/__tests__/clients.test.ts index 5bc780029..78f4f536b 100644 --- a/src/lib/documents/sources/__tests__/clients.test.ts +++ b/src/lib/documents/sources/__tests__/clients.test.ts @@ -171,8 +171,10 @@ describe("Paperless-ngx", () => { const list = calledUrls().find((u) => u.pathname.endsWith("/documents/")); expect(list?.searchParams.get("title__icontains")).toBe("blood"); expect(list?.searchParams.get("tags__id__all")).toBe("7"); - expect(list?.searchParams.get("created__date__gte")).toBe("2025-01-01"); - expect(list?.searchParams.get("created__date__lte")).toBe("2025-12-31"); + expect(list?.searchParams.get("created__gte")).toBe("2025-01-01"); + expect(list?.searchParams.get("created__lte")).toBe("2025-12-31"); + // API version 9 made `created` a date; the `__date` lookups are gone. + expect(list?.searchParams.has("created__date__gte")).toBe(false); expect(list?.searchParams.get("page")).toBe("2"); expect(list?.searchParams.get("page_size")).toBe("25"); @@ -311,6 +313,76 @@ describe("Paperless-ngx", () => { }); }); +describe("ids never shape a request path", () => { + it("refuses a Paperless id that is not a number, and dot segments, before dialling", async () => { + const client = sourceClient(PAPERLESS); + for (const id of ["..", ".", "41/../x", "abc", "41?x=1"]) { + const err = await failure(client.document(id)); + expect((err as DocumentSourceError).code, id).toBe("notFound"); + expect( + ((await failure(client.download(id, 10))) as DocumentSourceError).code, + ).toBe("notFound"); + } + expect(safeFetch).not.toHaveBeenCalled(); + }); + + it("refuses a Papra id or organization id outside [A-Za-z0-9_-]", async () => { + for (const id of ["..", "doc/1", "doc%2F1", "doc 1"]) { + await failure(sourceClient(PAPRA).document(id)); + } + expect(() => + sourceClient({ ...PAPRA, organizationId: "../admin" }), + ).toThrow(DocumentSourceError); + expect(safeFetch).not.toHaveBeenCalled(); + }); + + it("leaves a result with an unusable id out of the list", async () => { + serve({ + "/paperless/api/tags/": () => json({ results: [], next: null }), + "/paperless/api/documents/": () => + json({ + next: null, + results: [ + { id: "..", title: "Odd", created: "2025-01-01", tags: [] }, + { id: 5, title: "Fine", created: "2025-01-01", tags: [] }, + ], + }), + }); + const result = await sourceClient(PAPERLESS).search({ + q: "", + tagId: null, + from: null, + to: null, + page: 1, + }); + expect(result.items.map((i) => i.sourceId)).toEqual(["5"]); + }); +}); + +describe("Paperless-ngx tags", () => { + it("pages through tags beyond the first 250", async () => { + serve({ + "/paperless/api/tags/": (url) => { + const page = Number(url.searchParams.get("page")); + const results = Array.from( + { length: page === 1 ? 250 : 3 }, + (_, i) => ({ + id: page * 1000 + i, + name: `t${page}-${i}`, + }), + ); + return json({ results, next: page === 1 ? "more" : null }); + }, + }); + const tags = await sourceClient(PAPERLESS).tags(); + expect(tags).toHaveLength(253); + expect(calledUrls().map((u) => u.searchParams.get("page"))).toEqual([ + "1", + "2", + ]); + }); +}); + describe("Papra", () => { const tags = { tags: [ diff --git a/src/lib/documents/sources/clients.ts b/src/lib/documents/sources/clients.ts index 7bca6ed92..a5d1cfa85 100644 --- a/src/lib/documents/sources/clients.ts +++ b/src/lib/documents/sources/clients.ts @@ -29,6 +29,8 @@ import { import { DOCUMENT_PICKER_MAX_TAGS, DOCUMENT_PICKER_PAGE_SIZE, + isSourceDocumentId, + PAPRA_ID, type DocumentPickerSystem, type DocumentSourceTagDto, } from "./types"; @@ -64,6 +66,11 @@ export interface SourceDocumentMeta { export interface SourceClient { readonly system: DocumentPickerSystem; + /** + * The instance this client talks to, as the normalised origin every source + * key it produces carries (`sourceInstance`). + */ + readonly instance: string; test(): Promise; tags(): Promise; search( @@ -84,6 +91,16 @@ export interface SourceConnection { token: string; } +/** Paperless tags are read this many per page. */ +const TAG_PAGE_SIZE = 250; + +/** Refuse an id that is not one the system issues, before it reaches a URL. */ +function requireId(system: DocumentPickerSystem, id: string): void { + if (!isSourceDocumentId(system, id)) { + throw new DocumentSourceError("notFound"); + } +} + // ── Defensive readers ────────────────────────────────────────────────────── type Json = Record; @@ -181,26 +198,36 @@ function paperless(connection: SourceConnection): SourceClient { } async function listTags(): Promise { - const query = new URLSearchParams({ - page_size: String(DOCUMENT_PICKER_MAX_TAGS), - ordering: "name", - }); - const { body, response } = await sourceJson(target, `/api/tags/?${query}`); - checkVersion(response); - const page = asRecord(body); - if (!page) throw shapeError(); + // Paged by number rather than by following `next`: the link is the + // server's to write, the page number is ours. const tags: DocumentSourceTagDto[] = []; - for (const raw of asArray(page.results)) { - const tag = asRecord(raw); - const id = asText(tag?.id); - const name = asText(tag?.name); - if (id && name) tags.push({ id, name }); + for (let pageNo = 1; tags.length < DOCUMENT_PICKER_MAX_TAGS; pageNo += 1) { + const query = new URLSearchParams({ + page: String(pageNo), + page_size: String(TAG_PAGE_SIZE), + ordering: "name", + }); + const { body, response } = await sourceJson( + target, + `/api/tags/?${query}`, + ); + checkVersion(response); + const page = asRecord(body); + if (!page) throw shapeError(); + for (const raw of asArray(page.results)) { + const tag = asRecord(raw); + const id = asText(tag?.id); + const name = asText(tag?.name); + if (id && name) tags.push({ id, name }); + } + if (typeof page.next !== "string" || !page.next) break; } - return tags; + return tags.slice(0, DOCUMENT_PICKER_MAX_TAGS); } return { system: "PAPERLESS", + instance: connection.origin, async test() { const { body, response } = await sourceJson( @@ -223,8 +250,10 @@ function paperless(connection: SourceConnection): SourceClient { }); if (params.q) query.set("title__icontains", params.q); if (params.tagId) query.set("tags__id__all", params.tagId); - if (params.from) query.set("created__date__gte", params.from); - if (params.to) query.set("created__date__lte", params.to); + // API version 9 made `created` a plain date, so the range filters on + // it directly (`created__gte` / `created__lte`), both ends inclusive. + if (params.from) query.set("created__gte", params.from); + if (params.to) query.set("created__lte", params.to); let page: Json | null; let names: Map; @@ -250,7 +279,7 @@ function paperless(connection: SourceConnection): SourceClient { for (const raw of asArray(page.results)) { const doc = asRecord(raw); const id = asText(doc?.id); - if (!doc || !id) continue; + if (!doc || !id || !isSourceDocumentId("PAPERLESS", id)) continue; const tagIds = asArray(doc.tags) .map(asText) .filter((t): t is string => t !== null); @@ -275,6 +304,7 @@ function paperless(connection: SourceConnection): SourceClient { }, async document(sourceId) { + requireId("PAPERLESS", sourceId); const { body, response } = await sourceJson( target, `/api/documents/${encodeURIComponent(sourceId)}/?fields=id,title,created,added,original_file_name`, @@ -290,7 +320,8 @@ function paperless(connection: SourceConnection): SourceClient { }; }, - download(sourceId, maxFileBytes) { + async download(sourceId, maxFileBytes) { + requireId("PAPERLESS", sourceId); // The original, not the archived PDF/A: the same bytes a Paperless // workflow or the import script sends, so they dedup against each other. return sourceDownload( @@ -310,7 +341,7 @@ function papraTagTerm(name: string): string { } function papra(connection: SourceConnection): SourceClient { - if (!connection.organizationId) { + if (!connection.organizationId || !PAPRA_ID.test(connection.organizationId)) { throw new DocumentSourceError("notFound"); } const org = `/api/organizations/${encodeURIComponent(connection.organizationId)}`; @@ -345,6 +376,7 @@ function papra(connection: SourceConnection): SourceClient { return { system: "PAPRA", + instance: connection.origin, async test() { const { body } = await sourceJson( @@ -381,7 +413,7 @@ function papra(connection: SourceConnection): SourceClient { for (const raw of documents) { const doc = asRecord(raw); const id = asText(doc?.id); - if (!doc || !id) continue; + if (!doc || !id || !isSourceDocumentId("PAPRA", id)) continue; const tags = asArray(doc.tags) .map(asRecord) .filter((t): t is Json => t !== null); @@ -417,6 +449,7 @@ function papra(connection: SourceConnection): SourceClient { }, async document(sourceId) { + requireId("PAPRA", sourceId); const { body } = await sourceJson( target, `${org}/documents/${encodeURIComponent(sourceId)}`, @@ -435,7 +468,8 @@ function papra(connection: SourceConnection): SourceClient { }; }, - download(sourceId, maxFileBytes) { + async download(sourceId, maxFileBytes) { + requireId("PAPRA", sourceId); return sourceDownload( download, `${org}/documents/${encodeURIComponent(sourceId)}/file`, diff --git a/src/lib/documents/sources/route-support.ts b/src/lib/documents/sources/route-support.ts index 295944b8c..a1b9aba86 100644 --- a/src/lib/documents/sources/route-support.ts +++ b/src/lib/documents/sources/route-support.ts @@ -39,12 +39,13 @@ import { * Each route calls `requireAuth()` itself, in its own module, so the sharing * guard's scan sees every one of them as a bare, owner-only route rather than * a helper it cannot follow. This adds the rest: a cookie transport, the - * documents module, and (unless `needsList` is false, which only the status - * read passes) the operator's list. + * documents module and the operator's list. The status read and the delete + * pass `needsList: false, needsModule: false`: they touch only the person's + * own row and never the network. */ export async function admitDocumentSourceCaller( auth: AuthContext, - options: { needsList?: boolean } = {}, + options: { needsList?: boolean; needsModule?: boolean } = {}, ): Promise { if (auth.authMethod !== "cookie") { return apiError( @@ -53,8 +54,13 @@ export async function admitDocumentSourceCaller( { errorCode: "documents.sources.browserOnly" }, ); } - const gate = await requireModuleEnabled(auth.user.id, "inboundDocuments"); - if (!gate.enabled) return gate.response; + // Reading and removing one's own connection needs neither the module nor + // the list: a person who switched Documents off, or whose operator removed + // the list, must still be able to see and delete the stored token. + if (options.needsModule !== false) { + const gate = await requireModuleEnabled(auth.user.id, "inboundDocuments"); + if (!gate.enabled) return gate.response; + } if (options.needsList !== false && !documentSourcesAvailable()) { return sourceErrorResponse("unavailable"); } diff --git a/src/lib/documents/sources/types.ts b/src/lib/documents/sources/types.ts index 1e6cdf774..6dd8ddf35 100644 --- a/src/lib/documents/sources/types.ts +++ b/src/lib/documents/sources/types.ts @@ -38,8 +38,26 @@ export const DOCUMENT_PICKER_MAX_PAGE = 500; export const DOCUMENT_PICKER_QUERY_MAX = 200; /** Documents the sheet imports in one run. */ export const DOCUMENT_PICKER_MAX_SELECTION = 25; -/** Tags offered in the filter. */ -export const DOCUMENT_PICKER_MAX_TAGS = 250; +/** Tags offered in the filter (read in pages of 250). */ +export const DOCUMENT_PICKER_MAX_TAGS = 1000; + +/** Paperless-ngx document ids are integers. */ +export const PAPERLESS_ID = /^\d{1,18}$/; +/** Papra document and organization ids: letters, digits, `_` and `-`. */ +export const PAPRA_ID = /^[A-Za-z0-9_-]{1,128}$/; + +/** + * Is `id` a document id `system` could have issued? Checked before an id is + * put into a request path, so nothing a request carries can add a path + * segment, a query or a `..` to the URL built from it. + */ +export function isSourceDocumentId( + system: DocumentPickerSystem, + id: string, +): boolean { + if (id === "." || id === "..") return false; + return (system === "PAPERLESS" ? PAPERLESS_ID : PAPRA_ID).test(id); +} /** A connection as every route may show it: never the token. */ export interface DocumentSourceConnectionDto { diff --git a/src/lib/export/records-backup.ts b/src/lib/export/records-backup.ts index d924c8c59..2bb4cd278 100644 --- a/src/lib/export/records-backup.ts +++ b/src/lib/export/records-backup.ts @@ -177,6 +177,8 @@ export interface DocumentBackupEntry { */ sourceSystem?: string | null; sourceId?: string | null; + /** v1.39.3 — the source instance (origin); absent in older files. */ + sourceInstance?: string | null; /** Canonical DR only: the import's hold on automatic AI reading. */ aiReadDeferred?: boolean; } @@ -309,6 +311,7 @@ export async function buildRecordsBackupSection( lastIndexOutcome: true, sourceSystem: true, sourceId: true, + sourceInstance: true, aiReadDeferred: true, createdAt: true, updatedAt: true, @@ -330,6 +333,7 @@ export async function buildRecordsBackupSection( summaryEncrypted: true, sourceSystem: true, sourceId: true, + sourceInstance: true, createdAt: true, }, }); @@ -545,6 +549,7 @@ export async function buildRecordsBackupSection( : null, sourceSystem: d.sourceSystem, sourceId: d.sourceId, + sourceInstance: d.sourceInstance, createdAt: d.createdAt.toISOString(), }; diff --git a/src/lib/export/restore-backup.ts b/src/lib/export/restore-backup.ts index 3e60c6b51..d10100e78 100644 --- a/src/lib/export/restore-backup.ts +++ b/src/lib/export/restore-backup.ts @@ -30,6 +30,7 @@ import { Buffer } from "node:buffer"; import { prisma, toJson } from "@/lib/db"; +import { normaliseSourceInstance } from "@/lib/validations/inbound-documents"; import { auditLog } from "@/lib/auth/audit"; import { isStoredBackupReadError, @@ -2057,6 +2058,14 @@ export async function restoreBackup( sourceId: document.sourceSystem ? (document.sourceId ?? null) : null, + // A file written before v1.39.3 has no instance: the key then + // matches any instance of its system, as it did when stored. + sourceInstance: + document.sourceSystem && document.sourceId + ? document.sourceInstance + ? normaliseSourceInstance(document.sourceInstance) + : null + : null, createdAt: new Date(document.createdAt!), updatedAt: new Date(document.updatedAt!), })), diff --git a/src/lib/jobs/document-purge.ts b/src/lib/jobs/document-purge.ts index 40106a370..d98e9a988 100644 --- a/src/lib/jobs/document-purge.ts +++ b/src/lib/jobs/document-purge.ts @@ -58,9 +58,12 @@ export async function purgeExpiredDocumentTombstones( userId: true, sourceSystem: true, sourceId: true, + sourceInstance: true, // Further keys the import answered with this document (same bytes). // They cascade away with the row, so they are copied here first. - sourceAliases: { select: { sourceSystem: true, sourceId: true } }, + sourceAliases: { + select: { sourceSystem: true, sourceId: true, sourceInstance: true }, + }, }, }); const keys = keyed.flatMap((row) => [ @@ -70,6 +73,7 @@ export async function purgeExpiredDocumentTombstones( userId: row.userId, sourceSystem: row.sourceSystem, sourceId: row.sourceId, + sourceInstance: row.sourceInstance, }, ] : []), @@ -77,6 +81,7 @@ export async function purgeExpiredDocumentTombstones( userId: row.userId, sourceSystem: alias.sourceSystem, sourceId: alias.sourceId, + sourceInstance: alias.sourceInstance, })), ]); if (keys.length > 0) { diff --git a/src/lib/validations/backup.ts b/src/lib/validations/backup.ts index d9aa93bfe..c42b5b566 100644 --- a/src/lib/validations/backup.ts +++ b/src/lib/validations/backup.ts @@ -83,6 +83,7 @@ import { } from "@/lib/validations/emergency-profile"; import { DOCUMENT_SOURCE_ID_MAX, + DOCUMENT_SOURCE_INSTANCE_MAX, DOCUMENT_SOURCE_SYSTEMS, } from "@/lib/validations/inbound-documents"; import { REMINDER_EVENT_SOURCES } from "@/lib/measurement-reminders/satisfy"; @@ -1636,6 +1637,14 @@ const documentBackupSchema = z // restore cannot write a system the detail sheet has no name for. sourceSystem: z.enum(DOCUMENT_SOURCE_SYSTEMS).nullable().optional(), sourceId: z.string().max(DOCUMENT_SOURCE_ID_MAX).nullable().optional(), + // v1.39.3 — the source instance (an origin). Length-bounded here and + // normalised on restore; a value that is not an origin restores as null + // (matches any instance) rather than failing the restore. + sourceInstance: z + .string() + .max(DOCUMENT_SOURCE_INSTANCE_MAX) + .nullable() + .optional(), aiReadDeferred: z.boolean().optional(), summary: z.string().nullable().optional(), createdAt: isoDateTime.optional(), diff --git a/src/lib/validations/document-sources.ts b/src/lib/validations/document-sources.ts index 6997cb97d..d175c601c 100644 --- a/src/lib/validations/document-sources.ts +++ b/src/lib/validations/document-sources.ts @@ -10,6 +10,7 @@ import { DOCUMENT_PICKER_LINK_KINDS, DOCUMENT_PICKER_MAX_PAGE, DOCUMENT_PICKER_QUERY_MAX, + PAPRA_ID, } from "@/lib/documents/sources/types"; import { documentSourceIdSchema, @@ -37,9 +38,7 @@ export const documentSourceSaveSchema = z.object({ organizationId: z .string() .trim() - .min(1) - .max(128) - .regex(printable, "Expected printable characters without spaces") + .regex(PAPRA_ID, "Expected letters, digits, _ or -") .optional(), token: z .string() diff --git a/src/lib/validations/inbound-documents.ts b/src/lib/validations/inbound-documents.ts index ebc8aa030..afe29cd78 100644 --- a/src/lib/validations/inbound-documents.ts +++ b/src/lib/validations/inbound-documents.ts @@ -706,7 +706,41 @@ export const documentSourceIdSchema = z .trim() .min(1) .max(DOCUMENT_SOURCE_ID_MAX) - .regex(/^[\x21-\x7e]+$/u, "Expected printable characters without spaces"); + .regex(/^[\x21-\x7e]+$/u, "Expected printable characters without spaces") + // A path segment of its own name: an importer that builds a URL from the id + // must never be handed one that walks up the source's path. + .refine((v) => v !== "." && v !== "..", "Not a document id"); + +/** Longest source instance (an origin) a key may carry. */ +export const DOCUMENT_SOURCE_INSTANCE_MAX = 512; + +/** + * v1.39.3 — the instance of the source system a key belongs to, as the + * normalised origin (`scheme://host[:port]`) of the Paperless-ngx or Papra it + * came from, or null when the value is not one. A path, query or credentials + * in the value are dropped: two addresses on one origin are one instance. + */ +export function normaliseSourceInstance(value: string): string | null { + try { + const url = new URL(value.trim()); + if (url.protocol !== "http:" && url.protocol !== "https:") return null; + if (url.username || url.password) return null; + return url.origin; + } catch { + return null; + } +} + +/** The instance as a request field: validated, then normalised to its origin. */ +export const documentSourceInstanceSchema = z + .string() + .trim() + .min(1) + .max(DOCUMENT_SOURCE_INSTANCE_MAX) + .refine((v) => normaliseSourceInstance(v) !== null, { + message: "Expected the http(s) address of the source instance", + }) + .transform((v) => normaliseSourceInstance(v) as string); /** * A source key on its own — the query-string form the upload accepts ahead of @@ -715,6 +749,8 @@ export const documentSourceIdSchema = z export const documentSourceKeySchema = z.object({ sourceSystem: z.enum(DOCUMENT_SOURCE_SYSTEMS), sourceId: documentSourceIdSchema, + /** Optional: without it the key matches any instance of the system. */ + sourceInstance: documentSourceInstanceSchema.optional(), }); export const documentCreateSchema = z @@ -731,6 +767,11 @@ export const documentCreateSchema = z * not prose, and it is echoed on the detail sheet. */ sourceId: documentSourceIdSchema.optional(), + /** + * v1.39.3 — which instance of `sourceSystem` (its origin). Optional; a + * key without one matches any instance. + */ + sourceInstance: documentSourceInstanceSchema.optional(), /** * `defer` holds back automatic AI reading for this upload: the thumbnail * and a local text index still run, the summary and the lab staging do @@ -741,6 +782,10 @@ export const documentCreateSchema = z .refine((v) => v.sourceId === undefined || v.sourceSystem !== undefined, { path: ["sourceSystem"], message: "sourceId needs a sourceSystem", + }) + .refine((v) => v.sourceInstance === undefined || v.sourceId !== undefined, { + path: ["sourceInstance"], + message: "sourceInstance needs a sourceId", }); export type DocumentCreateInput = z.infer; diff --git a/tests/integration/document-sources.test.ts b/tests/integration/document-sources.test.ts index e729e7aed..0bcb842e6 100644 --- a/tests/integration/document-sources.test.ts +++ b/tests/integration/document-sources.test.ts @@ -56,6 +56,8 @@ vi.mock("@/lib/db-compat", () => ({ // ── The fake Paperless-ngx behind `safeFetch` ────────────────────────────── const ORIGIN = "http://paperless.lan:8000"; +/** A second Paperless-ngx the operator also listed; same ids, other documents. */ +const ORIGIN_B = "https://paperless-b.example.com"; const TOKEN = "paperless-token-0123456789-secret"; const PDF = (tag: string) => @@ -83,6 +85,10 @@ const DOCS = [ const fake = vi.hoisted(() => ({ calls: [] as Array<{ url: string; opts: Record }>, acceptToken: "", + /** Delay a download so two imports of one key overlap. */ + downloadDelayMs: 0, + /** Answer downloads with this status instead of the file. */ + downloadStatus: 200, })); vi.mock("@/lib/safe-fetch", async (importOriginal) => { @@ -113,6 +119,31 @@ vi.mock("@/lib/safe-fetch", async (importOriginal) => { return json({ results: [{ id: 7, name: "Health" }], next: null }); } if (path === "/api/documents/") { + // Tag 99: a server that ignores the tag filter, with the only match + // on its second page. + if (url.searchParams.get("tags__id__all") === "99") { + const page2 = url.searchParams.get("page") === "2"; + return json({ + next: page2 ? null : "next", + results: page2 + ? [ + { + id: 41, + title: "Blood test", + created: "2024-03-04", + tags: [99], + }, + ] + : [ + { + id: 42, + title: "Discharge letter", + created: "2023-11-20", + tags: [], + }, + ], + }); + } const q = ( url.searchParams.get("title__icontains") ?? "" ).toLowerCase(); @@ -126,8 +157,19 @@ vi.mock("@/lib/safe-fetch", async (importOriginal) => { const download = /^\/api\/documents\/(\d+)\/download\/$/.exec(path); if (download) { const doc = DOCS.find((d) => String(d.id) === download[1]); + if (fake.downloadDelayMs > 0) { + await new Promise((r) => setTimeout(r, fake.downloadDelayMs)); + } + if (fake.downloadStatus !== 200) { + return new Response("", { status: fake.downloadStatus }); + } + // Each instance holds different bytes under the same id. return doc - ? new Response(new Uint8Array(doc.bytes)) + ? new Response( + new Uint8Array( + Buffer.concat([doc.bytes, Buffer.from(`% ${url.origin}\n`)]), + ), + ) : new Response("", { status: 404 }); } const detail = /^\/api\/documents\/(\d+)\/$/.exec(path); @@ -259,7 +301,9 @@ beforeEach(async () => { headerJar.clear(); fake.calls.length = 0; fake.acceptToken = TOKEN; - process.env.DOCUMENT_SOURCE_ORIGINS = ORIGIN; + fake.downloadDelayMs = 0; + fake.downloadStatus = 200; + process.env.DOCUMENT_SOURCE_ORIGINS = `${ORIGIN},${ORIGIN_B}`; }); afterEach(() => { @@ -338,7 +382,7 @@ describe("saving a connection", () => { const read = await (await status()).text(); expect(read).not.toContain(TOKEN); - expect(JSON.parse(read).data.allowedOrigins).toEqual([ORIGIN]); + expect(JSON.parse(read).data.allowedOrigins).toEqual([ORIGIN, ORIGIN_B]); // A save without a token keeps the stored one. const again = await save({ baseUrl: ORIGIN }); @@ -495,7 +539,15 @@ describe("search and import", () => { // The same bytes, uploaded through the vault without a source key. const form = new FormData(); - form.append("file", new Blob([new Uint8Array(DOCS[1].bytes)]), "x.pdf"); + form.append( + "file", + new Blob([ + new Uint8Array( + Buffer.concat([DOCS[1].bytes, Buffer.from(`% ${ORIGIN}\n`)]), + ), + ]), + "x.pdf", + ); const { POST } = await import("@/app/api/documents/inbound/route"); const uploaded = await ( POST as unknown as (r: Request) => Promise @@ -520,6 +572,7 @@ describe("search and import", () => { where: { documentId: uploadedId, sourceSystem: "PAPERLESS", + sourceInstance: ORIGIN, sourceId: "42", }, }), @@ -583,3 +636,139 @@ describe("search and import", () => { expect((await disconnect()).status).toBe(200); }); }); + +describe("review follow-ups", () => { + it("never sends the stored token to a new origin", async () => { + await seedOwner(); + await connect(); + fake.calls.length = 0; + const moved = await save({ baseUrl: ORIGIN_B }); + expect(moved.status).toBe(422); + expect((await moved.json()).meta.errorCode).toBe( + "documents.sources.tokenRequired", + ); + expect(fake.calls).toHaveLength(0); + // A new path on the same origin keeps the token. + const samePlace = await save({ baseUrl: `${ORIGIN}/` }); + expect(samePlace.status).toBe(200); + expect(fake.calls.every((c) => c.url.startsWith(ORIGIN))).toBe(true); + }); + + it("keeps the same id on two instances apart, and lets a key without an instance match any", async () => { + await seedOwner(); + await connect(); + const prisma = getPrismaClient(); + + const first = await importDoc({ sourceId: "41" }); + expect(first.status).toBe(201); + const a = (await first.json()).data.documentId as string; + expect( + (await prisma.inboundDocument.findUniqueOrThrow({ where: { id: a } })) + .sourceInstance, + ).toBe(ORIGIN); + + // Switch the connection to the other Paperless-ngx. + expect((await save({ baseUrl: ORIGIN_B, token: TOKEN })).status).toBe(200); + const listed = (await (await search("q=blood")).json()).data.results[0]; + expect(listed).toMatchObject({ sourceId: "41", state: "new" }); + const second = await importDoc({ sourceId: "41" }); + expect(second.status).toBe(201); + const b = (await second.json()).data.documentId as string; + expect(b).not.toBe(a); + expect( + (await prisma.inboundDocument.findUniqueOrThrow({ where: { id: b } })) + .sourceInstance, + ).toBe(ORIGIN_B); + + // A v1.39.2 import of #42 carries no instance: it answers for any + // instance of its system, so the picker neither re-imports it nor + // forgets a deletion of it. + await prisma.inboundDocument.create({ + data: { + userId, + kind: "OTHER", + mimeType: "application/pdf", + byteSize: 1, + contentEncrypted: Buffer.from("x"), + contentSha256: "legacy-sha", + status: "STORED", + sourceSystem: "PAPERLESS", + sourceId: "42", + deletedAt: new Date(), + }, + }); + const legacy = (await (await search("q=discharge")).json()).data.results[0]; + expect(legacy).toMatchObject({ sourceId: "42", state: "deleted" }); + const refused = await importDoc({ sourceId: "42" }); + expect((await refused.json()).data.outcome).toBe("deleted"); + }); + + it("refuses an id that could leave the document path, before dialling", async () => { + await seedOwner(); + await connect(); + fake.calls.length = 0; + for (const sourceId of ["..", ".", "41/../../admin", "abc", "41?x=1"]) { + const res = await importDoc({ sourceId }); + expect(res.status, sourceId).toBe(422); + } + expect(fake.calls).toHaveLength(0); + }); + + it("downloads a key once when two imports of it overlap", async () => { + await seedOwner(); + await connect(); + fake.downloadDelayMs = 300; + const [one, two] = await Promise.all([ + importDoc({ sourceId: "41" }), + importDoc({ sourceId: "41" }), + ]); + const outcomes = [ + (await one.json()).data.outcome, + (await two.json()).data.outcome, + ].sort(); + expect(outcomes).toEqual(["duplicate", "imported"]); + expect(downloads()).toBe(1); + expect(await getPrismaClient().inboundDocument.count()).toBe(1); + }); + + it("does not charge the upload allowance for a fetch that failed", async () => { + await seedOwner(); + await connect(); + fake.downloadStatus = 500; + const res = await importDoc({ sourceId: "41" }); + expect(res.status).toBe(502); + expect( + await getPrismaClient().rateLimit.count({ + where: { key: `documents-upload:${userId}` }, + }), + ).toBe(0); + }); + + it("reads past a page the local re-check emptied", async () => { + await seedOwner(); + await connect(); + const res = await search("tag=99"); + expect(res.status).toBe(200); + const data = (await res.json()).data; + expect(data.page).toBe(2); + expect(data.results.map((r: { sourceId: string }) => r.sourceId)).toEqual([ + "41", + ]); + }); + + it("shows and deletes a connection with the module off and the list gone", async () => { + await seedOwner(); + await connect(); + delete process.env.DOCUMENT_SOURCE_ORIGINS; + await getPrismaClient().user.update({ + where: { id: userId }, + data: { modulePreferencesJson: { inboundDocuments: false } }, + }); + const read = (await (await status()).json()).data; + expect(read.available).toBe(false); + expect(read.connections).toHaveLength(1); + expect(read.connections[0].originAllowed).toBe(false); + expect((await disconnect()).status).toBe(200); + expect(await getPrismaClient().documentSourceConnection.count()).toBe(0); + }); +}); From 6a751bc51d027b993c55b705553fab17543d96b4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Sat, 26 Sep 2026 14:28:46 +0200 Subject: [PATCH 11/18] docs(api): publish sourceInstance on the upload and lookup routes, and the picker's review changes --- docs/api/openapi.yaml | 118 ++++++++++++++------- src/lib/openapi/routes/document-sources.ts | 10 +- src/lib/openapi/routes/documents.ts | 27 ++++- 3 files changed, 111 insertions(+), 44 deletions(-) diff --git a/docs/api/openapi.yaml b/docs/api/openapi.yaml index 91f77e5c8..5be68b466 100644 --- a/docs/api/openapi.yaml +++ b/docs/api/openapi.yaml @@ -15721,21 +15721,23 @@ paths: **Importing from another system (v1.39.2).** Optional `sourceSystem` (`PAPERLESS`, `PAPRA`, `OTHER`) and - `sourceId` (its id there, printable, up to 128 characters; needs `sourceSystem`) key the upload. A re-send with - a known key is a duplicate: 200 with the live row (`meta.duplicate: true`), or, when the owner DELETED that - document, 200 with `data.deleted: true` and `meta.deleted: true` and nothing stored; this holds after the 30-day - purge too. The key may also ride the query string (`?sourceSystem=…&sourceId=…`): a key the vault already holds - is then answered at once, the file is not stored or processed, and the request counts only toward the lookup - allowance shared with `GET /api/documents/inbound/source` (5000 an hour per token, per person for a session; 429 - past it), not toward the hourly upload limit. A key in both places must agree (else 422). Every other upload - counts toward the hourly upload limit, duplicates included. A key that is answered with an existing document by - its bytes is remembered for that document, so deleting it keeps that key deleted too; one document takes at most - 20 such keys, beyond which the upload is refused 409 (`documents.inbound.sourceAliasLimit`). `OTHER` is one - shared namespace per account: two different systems both sent as `OTHER` must not reuse each other's ids. - `aiRead=defer` holds back automatic AI reading for this upload: the thumbnail and a local text index still run, - the summary and lab staging do not, and the document stays out of every automatic AI path (the summary catch-up, - the automatic index, Index all for search) until the person reads it with AI or generates its summary. Absent, - behaviour is unchanged. + `sourceId` (its id there, printable, up to 128 characters, never `.` or `..`; needs `sourceSystem`) key the + upload, optionally with `sourceInstance` (v1.39.3: the address of the Paperless-ngx or Papra it came from, + normalised to its origin; without it the key matches any instance of the system). A re-send with a known key is + a duplicate: 200 with the live row (`meta.duplicate: true`), or, when the owner DELETED that document, 200 with + `data.deleted: true` and `meta.deleted: true` and nothing stored; this holds after the 30-day purge too. The key + may also ride the query string (`?sourceSystem=…&sourceId=…`): a key the vault already holds is then answered at + once, the file is not stored or processed, and the request counts only toward the lookup allowance shared with + `GET /api/documents/inbound/source` (5000 an hour per token, per person for a session; 429 past it), not toward + the hourly upload limit. A key in both places must agree (else 422). Every other upload counts toward the hourly + upload limit, duplicates included. A key that is answered with an existing document by its bytes is remembered + for that document, so deleting it keeps that key deleted too; one document takes at most 20 such keys, beyond + which the upload is refused 409 (`documents.inbound.sourceAliasLimit`). `OTHER` is one shared namespace per + account: two different systems both sent as `OTHER` must not reuse each other's ids. `aiRead=defer` holds back + automatic AI reading for this upload: the thumbnail and a local text index still run, the summary and lab + staging do not, and the document stays out of every automatic AI path (the summary catch-up, the automatic + index, Index all for search) until the person reads it with AI or generates its summary. Absent, behaviour is + unchanged. **Narrow token.** Also reachable with a `documents:write` Bearer (minted at `POST /api/tokens/documents`), which @@ -15767,6 +15769,16 @@ paths: type: string maxLength: 128 description: Import source key, second half. Both halves or neither; half a key is 422. + - name: sourceInstance + in: query + required: false + schema: + type: string + maxLength: 512 + description: "Optional, with a key. The instance of `sourceSystem` the id belongs to, as its http(s) address; normalised + to the origin (`scheme://host[:port]`). The same id in two instances is two documents. Optional: a key + stored without an instance (every upload before v1.39.3) matches any instance of its system, and a key sent + without one matches any stored instance. If also sent in the form it must agree." requestBody: required: true content: @@ -15821,6 +15833,13 @@ paths: documents. type: string maxLength: 128 + sourceInstance: + description: "Optional; requires `sourceId`. The instance of `sourceSystem` the id belongs to, as its http(s) address; + normalised to the origin (`scheme://host[:port]`). The same id in two instances is two documents. + Optional: a key stored without an instance (every upload before v1.39.3) matches any instance of its + system, and a key sent without one matches any stored instance." + type: string + maxLength: 512 aiRead: description: Optional. `defer` holds back automatic AI reading for this upload (local index and thumbnail only). type: string @@ -15896,6 +15915,16 @@ paths: schema: type: string maxLength: 128 + - name: sourceInstance + in: query + required: false + schema: + type: string + maxLength: 512 + description: "The instance of `sourceSystem` the id belongs to, as its http(s) address; normalised to the origin + (`scheme://host[:port]`). The same id in two instances is two documents. Optional: a key stored without an + instance (every upload before v1.39.3) matches any instance of its system, and a key sent without one + matches any stored instance." responses: "200": description: The answer for this key. @@ -20975,7 +21004,9 @@ paths: at least one origin), the listed origins, and the caller's own saved connections to Paperless-ngx or Papra. A connection never carries its token: `hasToken` is always true for a saved one, and no route returns the token in whole or in part. `originAllowed` is false once the operator removed the connection's origin from the list. No - request leaves the server. Cookie session only; not delegable." + request leaves the server. Needs neither the Documents module nor the list: with the list unset, `available` is + false and any connection still stored is listed (with `originAllowed: false`) so it can be removed. Cookie + session only; not delegable." security: &a52 - cookieAuth: [] responses: @@ -21004,9 +21035,11 @@ paths: description: "Saves or replaces the caller's connection to Paperless-ngx (`paperless`: base address and API token) or Papra (`papra`: base address, organization id, API key with `documents:read` and `tags:read`). The base address may carry a path; its origin must be listed in `DOCUMENT_SOURCE_ORIGINS`. The connection is tested live before - anything is stored. Leaving out `token` keeps the stored one (the first save must carry it). The token is stored - encrypted (AES-256-GCM) and never returned. Ten saves and tests a minute per person. Cookie session only; not - delegable." + anything is stored. Leaving out `token` keeps the stored one, but only when the new address is on the same + origin as the saved one: the first save and a move to another origin must carry it (422 + `documents.sources.tokenRequired`), so a stored token is never sent to an address it was not saved for. A Papra + organization id is letters, digits, `_` and `-`. The token is stored encrypted (AES-256-GCM) and never returned. + Ten saves and tests a minute per person. Cookie session only; not delegable." security: *a52 parameters: - &a54 @@ -21032,9 +21065,7 @@ paths: maxLength: 2048 organizationId: type: string - minLength: 1 - maxLength: 128 - pattern: ^[\x21-\x7e]+$ + pattern: ^[A-Za-z0-9_-]{1,128}$ token: type: string minLength: 1 @@ -21084,8 +21115,8 @@ paths: - Documents summary: Remove a connection to a document archive description: Deletes the caller's connection for this system, and its token with it. Works even after the operator - removed the origin from the list. `disconnected` is false when there was nothing to delete. Cookie session only; - not delegable. + removed the origin from the list, and with the Documents module switched off. `disconnected` is false when there + was nothing to delete. Cookie session only; not delegable. security: *a52 parameters: - *a54 @@ -21160,11 +21191,13 @@ paths: - Documents summary: Search a connected archive by name, tag and date description: "Searches the source 25 documents at a time. `q` matches the document's name (Paperless-ngx title, Papra - name); `tag` narrows to one of the source's tags by id; `from` / `to` to a date range, inclusive (Papra's range - is applied to each page after it arrives, so a page can hold fewer than 25). Each result carries what the vault - already holds under that source key: `new`, `imported` with the vault document's `documentId`, or `deleted` - (deleted in HealthLog; importing it again stores nothing). Nothing is stored. Sixty searches a minute, shared - with the tag read. Cookie session only; not delegable." + name); `tag` narrows to one of the source's tags by id; `from` / `to` to a date range, inclusive (Paperless-ngx + filters on `created__gte` / `created__lte`; Papra's range is applied to each page after it arrives). When a + filter applied after arrival empties a page, the route reads on, up to five source pages, and `page` in the + answer is the last source page read; ask for `page + 1` next. Each result carries what the vault already holds + under that source key: `new`, `imported` with the vault document's `documentId`, or `deleted` (deleted in + HealthLog; importing it again stores nothing). Nothing is stored. Sixty searches a minute, shared with the tag + read. Cookie session only; not delegable." security: *a52 parameters: - *a54 @@ -21218,15 +21251,19 @@ paths: tags: - Documents summary: Import one picked document - description: "Fetches one document from the source and stores it in the vault through the same path as an upload, with - `sourceSystem` / `sourceId` provenance. A source key the vault already holds is answered without a download: - `duplicate` for a live document (same id), `deleted` for one the person deleted in HealthLog, which is never - stored again. Bytes already in the vault under another key are `duplicate` too, and the key is remembered for - that document. `link` files the document against the record the picker was opened from (a condition, a visit or - a vaccination, which must be the caller's own and live; checked before anything is fetched), including a - duplicate. `kind` sets the document type (default OTHER). Automatic AI reading follows the person's own setting. - Counts against the person's upload allowance (60 an hour) and storage quota. 201 for a new document, 200 - otherwise. Cookie session only; not delegable." + description: "`sourceId` must be an id the system issues (Paperless-ngx: digits; Papra: letters, digits, `_`, `-`), else + 422 before anything is fetched. Fetches one document from the source and stores it in the vault through the same + path as an upload, with `sourceSystem` / `sourceId` / `sourceInstance` (the connection's origin) provenance; the + same id from another instance is another document. One import per key runs at a time; a concurrent second one + waits and then answers `duplicate`. The key check counts toward the lookup allowance, the fetch toward the + search allowance, and only a file that arrived toward the upload allowance. A source key the vault already holds + is answered without a download: `duplicate` for a live document (same id), `deleted` for one the person deleted + in HealthLog, which is never stored again. Bytes already in the vault under another key are `duplicate` too, and + the key is remembered for that document. `link` files the document against the record the picker was opened from + (a condition, a visit or a vaccination, which must be the caller's own and live; checked before anything is + fetched), including a duplicate. `kind` sets the document type (default OTHER). Automatic AI reading follows the + person's own setting. Counts against the person's upload allowance (60 an hour) and storage quota. 201 for a new + document, 200 otherwise. Cookie session only; not delegable." security: *a52 parameters: - *a54 @@ -35769,6 +35806,11 @@ components: - type: string maxLength: 128 - type: "null" + sourceInstance: + anyOf: + - type: string + maxLength: 512 + - type: "null" aiReadDeferred: type: boolean summary: diff --git a/src/lib/openapi/routes/document-sources.ts b/src/lib/openapi/routes/document-sources.ts index e25c0b813..53f4cfe6f 100644 --- a/src/lib/openapi/routes/document-sources.ts +++ b/src/lib/openapi/routes/document-sources.ts @@ -117,7 +117,7 @@ export const documentSourcePaths: NonNullable = { tags: ["Documents"], summary: "Document archives: availability and connections", description: - "Whether the operator enabled the document picker (`available`, true when `DOCUMENT_SOURCE_ORIGINS` lists at least one origin), the listed origins, and the caller's own saved connections to Paperless-ngx or Papra. A connection never carries its token: `hasToken` is always true for a saved one, and no route returns the token in whole or in part. `originAllowed` is false once the operator removed the connection's origin from the list. No request leaves the server. Cookie session only; not delegable.", + "Whether the operator enabled the document picker (`available`, true when `DOCUMENT_SOURCE_ORIGINS` lists at least one origin), the listed origins, and the caller's own saved connections to Paperless-ngx or Papra. A connection never carries its token: `hasToken` is always true for a saved one, and no route returns the token in whole or in part. `originAllowed` is false once the operator removed the connection's origin from the list. No request leaves the server. Needs neither the Documents module nor the list: with the list unset, `available` is false and any connection still stored is listed (with `originAllowed: false`) so it can be removed. Cookie session only; not delegable.", security: cookieOnly, responses: { "200": { @@ -141,7 +141,7 @@ export const documentSourcePaths: NonNullable = { tags: ["Documents"], summary: "Save a connection to a document archive", description: - "Saves or replaces the caller's connection to Paperless-ngx (`paperless`: base address and API token) or Papra (`papra`: base address, organization id, API key with `documents:read` and `tags:read`). The base address may carry a path; its origin must be listed in `DOCUMENT_SOURCE_ORIGINS`. The connection is tested live before anything is stored. Leaving out `token` keeps the stored one (the first save must carry it). The token is stored encrypted (AES-256-GCM) and never returned. Ten saves and tests a minute per person. Cookie session only; not delegable.", + "Saves or replaces the caller's connection to Paperless-ngx (`paperless`: base address and API token) or Papra (`papra`: base address, organization id, API key with `documents:read` and `tags:read`). The base address may carry a path; its origin must be listed in `DOCUMENT_SOURCE_ORIGINS`. The connection is tested live before anything is stored. Leaving out `token` keeps the stored one, but only when the new address is on the same origin as the saved one: the first save and a move to another origin must carry it (422 `documents.sources.tokenRequired`), so a stored token is never sent to an address it was not saved for. A Papra organization id is letters, digits, `_` and `-`. The token is stored encrypted (AES-256-GCM) and never returned. Ten saves and tests a minute per person. Cookie session only; not delegable.", security: cookieOnly, parameters: [systemParameter], requestBody: { @@ -176,7 +176,7 @@ export const documentSourcePaths: NonNullable = { tags: ["Documents"], summary: "Remove a connection to a document archive", description: - "Deletes the caller's connection for this system, and its token with it. Works even after the operator removed the origin from the list. `disconnected` is false when there was nothing to delete. Cookie session only; not delegable.", + "Deletes the caller's connection for this system, and its token with it. Works even after the operator removed the origin from the list, and with the Documents module switched off. `disconnected` is false when there was nothing to delete. Cookie session only; not delegable.", security: cookieOnly, parameters: [systemParameter], responses: { @@ -269,7 +269,7 @@ export const documentSourcePaths: NonNullable = { tags: ["Documents"], summary: "Search a connected archive by name, tag and date", description: - "Searches the source 25 documents at a time. `q` matches the document's name (Paperless-ngx title, Papra name); `tag` narrows to one of the source's tags by id; `from` / `to` to a date range, inclusive (Papra's range is applied to each page after it arrives, so a page can hold fewer than 25). Each result carries what the vault already holds under that source key: `new`, `imported` with the vault document's `documentId`, or `deleted` (deleted in HealthLog; importing it again stores nothing). Nothing is stored. Sixty searches a minute, shared with the tag read. Cookie session only; not delegable.", + "Searches the source 25 documents at a time. `q` matches the document's name (Paperless-ngx title, Papra name); `tag` narrows to one of the source's tags by id; `from` / `to` to a date range, inclusive (Paperless-ngx filters on `created__gte` / `created__lte`; Papra's range is applied to each page after it arrives). When a filter applied after arrival empties a page, the route reads on, up to five source pages, and `page` in the answer is the last source page read; ask for `page + 1` next. Each result carries what the vault already holds under that source key: `new`, `imported` with the vault document's `documentId`, or `deleted` (deleted in HealthLog; importing it again stores nothing). Nothing is stored. Sixty searches a minute, shared with the tag read. Cookie session only; not delegable.", security: cookieOnly, parameters: [ systemParameter, @@ -335,7 +335,7 @@ export const documentSourcePaths: NonNullable = { tags: ["Documents"], summary: "Import one picked document", description: - "Fetches one document from the source and stores it in the vault through the same path as an upload, with `sourceSystem` / `sourceId` provenance. A source key the vault already holds is answered without a download: `duplicate` for a live document (same id), `deleted` for one the person deleted in HealthLog, which is never stored again. Bytes already in the vault under another key are `duplicate` too, and the key is remembered for that document. `link` files the document against the record the picker was opened from (a condition, a visit or a vaccination, which must be the caller's own and live; checked before anything is fetched), including a duplicate. `kind` sets the document type (default OTHER). Automatic AI reading follows the person's own setting. Counts against the person's upload allowance (60 an hour) and storage quota. 201 for a new document, 200 otherwise. Cookie session only; not delegable.", + "`sourceId` must be an id the system issues (Paperless-ngx: digits; Papra: letters, digits, `_`, `-`), else 422 before anything is fetched. Fetches one document from the source and stores it in the vault through the same path as an upload, with `sourceSystem` / `sourceId` / `sourceInstance` (the connection's origin) provenance; the same id from another instance is another document. One import per key runs at a time; a concurrent second one waits and then answers `duplicate`. The key check counts toward the lookup allowance, the fetch toward the search allowance, and only a file that arrived toward the upload allowance. A source key the vault already holds is answered without a download: `duplicate` for a live document (same id), `deleted` for one the person deleted in HealthLog, which is never stored again. Bytes already in the vault under another key are `duplicate` too, and the key is remembered for that document. `link` files the document against the record the picker was opened from (a condition, a visit or a vaccination, which must be the caller's own and live; checked before anything is fetched), including a duplicate. `kind` sets the document type (default OTHER). Automatic AI reading follows the person's own setting. Counts against the person's upload allowance (60 an hour) and storage quota. 201 for a new document, 200 otherwise. Cookie session only; not delegable.", security: cookieOnly, parameters: [systemParameter], requestBody: { diff --git a/src/lib/openapi/routes/documents.ts b/src/lib/openapi/routes/documents.ts index d31ad375d..9bdbcf89e 100644 --- a/src/lib/openapi/routes/documents.ts +++ b/src/lib/openapi/routes/documents.ts @@ -24,6 +24,7 @@ import { inboundConfirmSchema, inboundFactEditSchema, DOCUMENT_SOURCE_ID_MAX, + DOCUMENT_SOURCE_INSTANCE_MAX, DOCUMENT_SOURCE_SYSTEMS, DOCUMENT_SUMMARY_STATES, INBOUND_DOCUMENT_KINDS, @@ -417,12 +418,20 @@ export const inboundDocumentPaths: NonNullable = { description: "Import source key, second half. Both halves or neither; half a key is 422.", }, + { + name: "sourceInstance", + in: "query", + required: false, + schema: { type: "string", maxLength: DOCUMENT_SOURCE_INSTANCE_MAX }, + description: + "Optional, with a key. The instance of `sourceSystem` the id belongs to, as its http(s) address; normalised to the origin (`scheme://host[:port]`). The same id in two instances is two documents. Optional: a key stored without an instance (every upload before v1.39.3) matches any instance of its system, and a key sent without one matches any stored instance. If also sent in the form it must agree.", + }, ], tags: ["Documents"], summary: "Store a document (no extraction)", description: 'STORE-ONLY upload. Stores the raw document ENCRYPTED at rest with `status: STORED` and runs NO extraction — provider-free, no AI consent / budget / egress. A file is always filable, even with no document-scan provider configured. `multipart/form-data`: a `file` plus optional `title`, `kind`, `documentDate` (YYYY-MM-DD), and repeated `episodeIds` form fields (pre-link to the caller\'s illness/condition episodes). Accepted types (magic-byte sniffed, never the wire Content-Type): PDF/JPEG/PNG/WebP/GIF render inline; Office (docx/xlsx/pptx/doc/xls/ppt), text/CSV/Markdown/RTF, TIFF, HEIC/HEIF, XML/JSON are stored verbatim and served download-only. HEIC is stored as-is but attachment-only — prefer transcoding to JPEG client-side for inline preview parity. Error contract: `413` with `meta.reason = "fileTooLarge"` (+ `maxFileBytes`) or `"quotaExceeded"` (+ `quotaBytes`, `usedBytes`); `415` with `meta.reason = "unsupportedType"`. A same-user duplicate (same bytes) returns 200 + `meta.duplicate: true` with the existing row — not an error. `Idempotency-Key` honoured. Read `GET /api/documents/inbound/usage` for the effective limits before offering an upload. AI extraction is a separate opt-in action — see `POST /api/documents/inbound/{id}/extract`.\n\n' + - "**Importing from another system (v1.39.2).** Optional `sourceSystem` (`PAPERLESS`, `PAPRA`, `OTHER`) and `sourceId` (its id there, printable, up to 128 characters; needs `sourceSystem`) key the upload. A re-send with a known key is a duplicate: 200 with the live row (`meta.duplicate: true`), or, when the owner DELETED that document, 200 with `data.deleted: true` and `meta.deleted: true` and nothing stored; this holds after the 30-day purge too. The key may also ride the query string (`?sourceSystem=…&sourceId=…`): a key the vault already holds is then answered at once, the file is not stored or processed, and the request counts only toward the lookup allowance shared with `GET /api/documents/inbound/source` (5000 an hour per token, per person for a session; 429 past it), not toward the hourly upload limit. A key in both places must agree (else 422). Every other upload counts toward the hourly upload limit, duplicates included. A key that is answered with an existing document by its bytes is remembered for that document, so deleting it keeps that key deleted too; one document takes at most 20 such keys, beyond which the upload is refused 409 (`documents.inbound.sourceAliasLimit`). `OTHER` is one shared namespace per account: two different systems both sent as `OTHER` must not reuse each other\'s ids. `aiRead=defer` holds back automatic AI reading for this upload: the thumbnail and a local text index still run, the summary and lab staging do not, and the document stays out of every automatic AI path (the summary catch-up, the automatic index, Index all for search) until the person reads it with AI or generates its summary. Absent, behaviour is unchanged.\n\n" + + "**Importing from another system (v1.39.2).** Optional `sourceSystem` (`PAPERLESS`, `PAPRA`, `OTHER`) and `sourceId` (its id there, printable, up to 128 characters, never `.` or `..`; needs `sourceSystem`) key the upload, optionally with `sourceInstance` (v1.39.3: the address of the Paperless-ngx or Papra it came from, normalised to its origin; without it the key matches any instance of the system). A re-send with a known key is a duplicate: 200 with the live row (`meta.duplicate: true`), or, when the owner DELETED that document, 200 with `data.deleted: true` and `meta.deleted: true` and nothing stored; this holds after the 30-day purge too. The key may also ride the query string (`?sourceSystem=…&sourceId=…`): a key the vault already holds is then answered at once, the file is not stored or processed, and the request counts only toward the lookup allowance shared with `GET /api/documents/inbound/source` (5000 an hour per token, per person for a session; 429 past it), not toward the hourly upload limit. A key in both places must agree (else 422). Every other upload counts toward the hourly upload limit, duplicates included. A key that is answered with an existing document by its bytes is remembered for that document, so deleting it keeps that key deleted too; one document takes at most 20 such keys, beyond which the upload is refused 409 (`documents.inbound.sourceAliasLimit`). `OTHER` is one shared namespace per account: two different systems both sent as `OTHER` must not reuse each other\'s ids. `aiRead=defer` holds back automatic AI reading for this upload: the thumbnail and a local text index still run, the summary and lab staging do not, and the document stays out of every automatic AI path (the summary catch-up, the automatic index, Index all for search) until the person reads it with AI or generates its summary. Absent, behaviour is unchanged.\n\n" + "**Narrow token.** Also reachable with a `documents:write` Bearer (minted at `POST /api/tokens/documents`), which reaches only this route and the source-key lookup `GET /api/documents/inbound/source`. Such a caller draws on a bucket of its own, keyed on the token (default 120 an hour, `DOCUMENT_UPLOAD_LIMIT_PER_HOUR`, clamped 1-1000; the 429 carries `Retry-After` and `X-RateLimit-*`), and gets a `DocumentUploadReceipt` instead of the stored row; its 413 `quotaExceeded` carries no `quotaBytes` / `usedBytes`. The response is an untagged union (row or receipt) on purpose: a discriminator would need a new field on the row every existing client decodes. A cookie or wildcard caller keeps the 60-an-hour per-user bucket and the full row. With the vault module off the answer is 403 `module.disabled` for every caller.", requestBody: { required: true, @@ -461,6 +470,14 @@ export const inboundDocumentPaths: NonNullable = { description: "Optional. The document's id in `sourceSystem`; requires it. Unique per account across live and deleted documents.", }), + sourceInstance: z + .string() + .max(DOCUMENT_SOURCE_INSTANCE_MAX) + .optional() + .meta({ + description: + "Optional; requires `sourceId`. The instance of `sourceSystem` the id belongs to, as its http(s) address; normalised to the origin (`scheme://host[:port]`). The same id in two instances is two documents. Optional: a key stored without an instance (every upload before v1.39.3) matches any instance of its system, and a key sent without one matches any stored instance.", + }), aiRead: z.enum(["defer"]).optional().meta({ description: "Optional. `defer` holds back automatic AI reading for this upload (local index and thumbnail only).", @@ -527,6 +544,14 @@ export const inboundDocumentPaths: NonNullable = { required: true, schema: { type: "string", maxLength: DOCUMENT_SOURCE_ID_MAX }, }, + { + name: "sourceInstance", + in: "query", + required: false, + schema: { type: "string", maxLength: DOCUMENT_SOURCE_INSTANCE_MAX }, + description: + "The instance of `sourceSystem` the id belongs to, as its http(s) address; normalised to the origin (`scheme://host[:port]`). The same id in two instances is two documents. Optional: a key stored without an instance (every upload before v1.39.3) matches any instance of its system, and a key sent without one matches any stored instance.", + }, ], responses: { "200": { From 6534c3f9e8887c0db71e5e717289ab9b2e3af3de Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Sat, 26 Sep 2026 14:32:46 +0200 Subject: [PATCH 12/18] fix(documents): picker and settings card follow-ups The settings card stays visible, read-only, while a connection is stored but the operator list is gone or Documents is off, so the token can still be removed; a new address asks for the token again. The picker clears a selection when the search or a filter changes, says when the 25-document limit is reached, steps back the rows a finished run did not touch and offers Pick more, shows an empty state, and announces each row's result to screen readers. Both system switches are radio groups with one tab stop and arrow-key movement. Strings in all seven languages. --- messages/de.json | 15 +- messages/en.json | 15 +- messages/es.json | 15 +- messages/fr.json | 15 +- messages/it.json | 15 +- messages/ko.json | 15 +- messages/pl.json | 15 +- .../__tests__/document-sources-ui.test.tsx | 47 ++- .../sources/document-source-picker.tsx | 216 ++++++++++---- .../documents/sources/segmented-choice.tsx | 110 +++++++ .../documents/sources/use-document-sources.ts | 37 ++- .../integrations/document-sources-card.tsx | 280 ++++++++++-------- 12 files changed, 567 insertions(+), 228 deletions(-) create mode 100644 src/components/documents/sources/segmented-choice.tsx diff --git a/messages/de.json b/messages/de.json index b0b80d39f..b0426eec2 100644 --- a/messages/de.json +++ b/messages/de.json @@ -6713,7 +6713,7 @@ "organizationId": "Organisations-ID", "organizationHelp": "Der Teil nach /organizations/ in der Adresszeile von Papra.", "token": "API-Token", - "tokenSaved": "Gespeichert. Leer lassen, um es zu behalten.", + "tokenSaved": "Für diese Adresse gespeichert. Leer lassen, um es zu behalten.", "tokenHelpPaperless": "In Paperless-ngx: Mein Profil, API-Token.", "tokenHelpPapra": "In Papra: ein API-Schlüssel mit documents:read und tags:read.", "save": "Speichern und testen", @@ -6723,7 +6723,10 @@ "testOk": "Das Archiv hat in {latency} ms geantwortet.", "disconnect": "Trennen", "disconnectTitle": "{name} trennen?", - "disconnectDescription": "HealthLog vergisst Adresse und Token. Bereits importierte Dokumente bleiben in HealthLog." + "disconnectDescription": "HealthLog vergisst Adresse und Token. Bereits importierte Dokumente bleiben in HealthLog.", + "unavailableExplainer": "Dieser Server erlaubt keine Dokumentarchive mehr, die Verbindung unten ist daher nicht nutzbar. Du kannst sie weiterhin entfernen.", + "moduleOffExplainer": "Dokumente sind ausgeschaltet, die Verbindung unten wird daher nicht genutzt. Du kannst sie weiterhin entfernen.", + "tokenNewAddress": "Für eine neue Adresse wird das API-Token erneut gebraucht." } }, "admin": { @@ -10223,7 +10226,7 @@ "browserOnly": "Dokumentarchive lassen sich nur im Browser verwenden.", "invalidAddress": "Gib die vollständige Adresse ein, beginnend mit https:// oder http://.", "organizationRequired": "Gib die Organisations-ID aus Papra ein.", - "tokenRequired": "Gib das API-Token ein.", + "tokenRequired": "Gib das API-Token ein. Eine neue Adresse braucht es erneut.", "fileTooLarge": "Die Datei ist größer, als HealthLog annimmt.", "quotaExceeded": "Dein Dokumentenspeicher ist voll.", "fileType": "HealthLog nimmt diesen Dateityp nicht an.", @@ -10231,7 +10234,11 @@ "uploadBusy": "Andere Uploads laufen noch. Versuche es gleich noch einmal.", "moduleDisabled": "Dokumente sind in den Einstellungen ausgeschaltet.", "generic": "Etwas ist schiefgelaufen. Versuche es erneut." - } + }, + "pickMore": "Weitere auswählen", + "finishedHint": "Suche erneut oder ändere einen Filter, um weitere Dokumente auszuwählen.", + "maxReached": "Du kannst höchstens {max} Dokumente auf einmal importieren.", + "noMatchHint": "Versuche einen Teil des Namens oder setze Tag und Datum zurück." } }, "selfDescription": { diff --git a/messages/en.json b/messages/en.json index 324370d50..f42ec1e19 100644 --- a/messages/en.json +++ b/messages/en.json @@ -6713,7 +6713,7 @@ "organizationId": "Organization id", "organizationHelp": "The part after /organizations/ in the Papra address bar.", "token": "API token", - "tokenSaved": "Saved. Leave empty to keep it.", + "tokenSaved": "Saved for this address. Leave empty to keep it.", "tokenHelpPaperless": "In Paperless-ngx: My Profile, API token.", "tokenHelpPapra": "In Papra: an API key with documents:read and tags:read.", "save": "Save and test", @@ -6723,7 +6723,10 @@ "testOk": "The archive answered in {latency} ms.", "disconnect": "Disconnect", "disconnectTitle": "Disconnect {name}?", - "disconnectDescription": "HealthLog forgets the address and the token. Documents you already imported stay in HealthLog." + "disconnectDescription": "HealthLog forgets the address and the token. Documents you already imported stay in HealthLog.", + "unavailableExplainer": "This server no longer allows document archives, so the connection below cannot be used. You can still remove it.", + "moduleOffExplainer": "Documents are switched off, so the connection below is not used. You can still remove it.", + "tokenNewAddress": "A new address needs the API token again." } }, "admin": { @@ -10223,7 +10226,7 @@ "browserOnly": "Document archives can only be used in the browser.", "invalidAddress": "Enter the full address, starting with https:// or http://.", "organizationRequired": "Enter the Papra organization id.", - "tokenRequired": "Enter the API token.", + "tokenRequired": "Enter the API token. A new address needs it again.", "fileTooLarge": "The file is larger than HealthLog accepts.", "quotaExceeded": "Your document storage is full.", "fileType": "HealthLog does not accept this file type.", @@ -10231,7 +10234,11 @@ "uploadBusy": "Other uploads are still running. Try again in a moment.", "moduleDisabled": "Documents are switched off in Settings.", "generic": "Something went wrong. Try again." - } + }, + "pickMore": "Pick more", + "finishedHint": "Search again or change a filter to pick more documents.", + "maxReached": "You can import up to {max} documents at a time.", + "noMatchHint": "Try part of the name, or clear the tag and dates." } }, "selfDescription": { diff --git a/messages/es.json b/messages/es.json index f367a4433..6f9e5127b 100644 --- a/messages/es.json +++ b/messages/es.json @@ -6713,7 +6713,7 @@ "organizationId": "Id de organización", "organizationHelp": "La parte que sigue a /organizations/ en la barra de direcciones de Papra.", "token": "Token de la API", - "tokenSaved": "Guardado. Déjalo vacío para conservarlo.", + "tokenSaved": "Guardado para esta dirección. Déjalo vacío para conservarlo.", "tokenHelpPaperless": "En Paperless-ngx: Mi perfil, token de la API.", "tokenHelpPapra": "En Papra: una clave de API con documents:read y tags:read.", "save": "Guardar y probar", @@ -6723,7 +6723,10 @@ "testOk": "El archivo respondió en {latency} ms.", "disconnect": "Desconectar", "disconnectTitle": "¿Desconectar {name}?", - "disconnectDescription": "HealthLog olvida la dirección y el token. Los documentos ya importados se quedan en HealthLog." + "disconnectDescription": "HealthLog olvida la dirección y el token. Los documentos ya importados se quedan en HealthLog.", + "unavailableExplainer": "Este servidor ya no permite archivos de documentos, así que la conexión de abajo no se puede usar. Aún puedes eliminarla.", + "moduleOffExplainer": "Los documentos están desactivados, así que la conexión de abajo no se usa. Aún puedes eliminarla.", + "tokenNewAddress": "Una dirección nueva necesita de nuevo el token de la API." } }, "admin": { @@ -10223,7 +10226,7 @@ "browserOnly": "Los archivos de documentos solo se pueden usar en el navegador.", "invalidAddress": "Introduce la dirección completa, empezando por https:// o http://.", "organizationRequired": "Introduce el id de organización de Papra.", - "tokenRequired": "Introduce el token de la API.", + "tokenRequired": "Introduce el token de la API. Una dirección nueva lo necesita de nuevo.", "fileTooLarge": "El archivo supera el tamaño que acepta HealthLog.", "quotaExceeded": "Tu almacenamiento de documentos está lleno.", "fileType": "HealthLog no acepta este tipo de archivo.", @@ -10231,7 +10234,11 @@ "uploadBusy": "Aún hay otras subidas en curso. Inténtalo en un momento.", "moduleDisabled": "Los documentos están desactivados en Ajustes.", "generic": "Algo salió mal. Inténtalo de nuevo." - } + }, + "pickMore": "Elegir más", + "finishedHint": "Busca de nuevo o cambia un filtro para elegir más documentos.", + "maxReached": "Puedes importar como máximo {max} documentos a la vez.", + "noMatchHint": "Prueba con parte del nombre o quita la etiqueta y las fechas." } }, "selfDescription": { diff --git a/messages/fr.json b/messages/fr.json index 1037939c5..93f029e4c 100644 --- a/messages/fr.json +++ b/messages/fr.json @@ -6713,7 +6713,7 @@ "organizationId": "Identifiant d'organisation", "organizationHelp": "La partie après /organizations/ dans la barre d'adresse de Papra.", "token": "Jeton d'API", - "tokenSaved": "Enregistré. Laissez vide pour le conserver.", + "tokenSaved": "Enregistré pour cette adresse. Laissez vide pour le conserver.", "tokenHelpPaperless": "Dans Paperless-ngx : Mon profil, jeton d'API.", "tokenHelpPapra": "Dans Papra : une clé d'API avec documents:read et tags:read.", "save": "Enregistrer et tester", @@ -6723,7 +6723,10 @@ "testOk": "L'archive a répondu en {latency} ms.", "disconnect": "Déconnecter", "disconnectTitle": "Déconnecter {name} ?", - "disconnectDescription": "HealthLog oublie l'adresse et le jeton. Les documents déjà importés restent dans HealthLog." + "disconnectDescription": "HealthLog oublie l'adresse et le jeton. Les documents déjà importés restent dans HealthLog.", + "unavailableExplainer": "Ce serveur n'autorise plus les archives de documents ; la connexion ci-dessous ne peut donc pas servir. Vous pouvez toujours la supprimer.", + "moduleOffExplainer": "Les documents sont désactivés ; la connexion ci-dessous n'est donc pas utilisée. Vous pouvez toujours la supprimer.", + "tokenNewAddress": "Une nouvelle adresse demande de saisir à nouveau le jeton d'API." } }, "admin": { @@ -10223,7 +10226,7 @@ "browserOnly": "Les archives de documents ne s'utilisent que dans le navigateur.", "invalidAddress": "Saisissez l'adresse complète, commençant par https:// ou http://.", "organizationRequired": "Saisissez l'identifiant d'organisation Papra.", - "tokenRequired": "Saisissez le jeton d'API.", + "tokenRequired": "Saisissez le jeton d'API. Une nouvelle adresse le demande à nouveau.", "fileTooLarge": "Le fichier dépasse la taille acceptée par HealthLog.", "quotaExceeded": "Votre espace de documents est plein.", "fileType": "HealthLog n'accepte pas ce type de fichier.", @@ -10231,7 +10234,11 @@ "uploadBusy": "D'autres envois sont en cours. Réessayez dans un instant.", "moduleDisabled": "Les documents sont désactivés dans les réglages.", "generic": "Un problème est survenu. Réessayez." - } + }, + "pickMore": "En choisir d'autres", + "finishedHint": "Cherchez à nouveau ou changez un filtre pour choisir d'autres documents.", + "maxReached": "Vous pouvez importer au plus {max} documents à la fois.", + "noMatchHint": "Essayez une partie du nom, ou effacez l'étiquette et les dates." } }, "selfDescription": { diff --git a/messages/it.json b/messages/it.json index 1d4c709f7..23f268c99 100644 --- a/messages/it.json +++ b/messages/it.json @@ -6713,7 +6713,7 @@ "organizationId": "Id organizzazione", "organizationHelp": "La parte dopo /organizations/ nella barra degli indirizzi di Papra.", "token": "Token API", - "tokenSaved": "Salvato. Lascia vuoto per mantenerlo.", + "tokenSaved": "Salvato per questo indirizzo. Lascia vuoto per mantenerlo.", "tokenHelpPaperless": "In Paperless-ngx: Il mio profilo, token API.", "tokenHelpPapra": "In Papra: una chiave API con documents:read e tags:read.", "save": "Salva e prova", @@ -6723,7 +6723,10 @@ "testOk": "L'archivio ha risposto in {latency} ms.", "disconnect": "Scollega", "disconnectTitle": "Scollegare {name}?", - "disconnectDescription": "HealthLog dimentica indirizzo e token. I documenti già importati restano in HealthLog." + "disconnectDescription": "HealthLog dimentica indirizzo e token. I documenti già importati restano in HealthLog.", + "unavailableExplainer": "Questo server non consente più gli archivi di documenti, quindi il collegamento qui sotto non si può usare. Puoi comunque rimuoverlo.", + "moduleOffExplainer": "I documenti sono disattivati, quindi il collegamento qui sotto non viene usato. Puoi comunque rimuoverlo.", + "tokenNewAddress": "Un nuovo indirizzo richiede di nuovo il token API." } }, "admin": { @@ -10223,7 +10226,7 @@ "browserOnly": "Gli archivi di documenti si usano solo nel browser.", "invalidAddress": "Inserisci l'indirizzo completo, che inizia con https:// o http://.", "organizationRequired": "Inserisci l'id dell'organizzazione Papra.", - "tokenRequired": "Inserisci il token API.", + "tokenRequired": "Inserisci il token API. Un nuovo indirizzo lo richiede di nuovo.", "fileTooLarge": "Il file è più grande di quanto HealthLog accetti.", "quotaExceeded": "Lo spazio per i documenti è pieno.", "fileType": "HealthLog non accetta questo tipo di file.", @@ -10231,7 +10234,11 @@ "uploadBusy": "Altri caricamenti sono ancora in corso. Riprova tra un momento.", "moduleDisabled": "I documenti sono disattivati nelle impostazioni.", "generic": "Qualcosa è andato storto. Riprova." - } + }, + "pickMore": "Scegline altri", + "finishedHint": "Cerca di nuovo o cambia un filtro per scegliere altri documenti.", + "maxReached": "Puoi importare al massimo {max} documenti alla volta.", + "noMatchHint": "Prova con parte del nome, oppure azzera etichetta e date." } }, "selfDescription": { diff --git a/messages/ko.json b/messages/ko.json index 42c93bcf7..fef692073 100644 --- a/messages/ko.json +++ b/messages/ko.json @@ -6713,7 +6713,7 @@ "organizationId": "조직 ID", "organizationHelp": "Papra 주소 표시줄에서 /organizations/ 다음 부분입니다.", "token": "API 토큰", - "tokenSaved": "저장됨. 유지하려면 비워 두세요.", + "tokenSaved": "이 주소에 저장됨. 유지하려면 비워 두세요.", "tokenHelpPaperless": "Paperless-ngx: 내 프로필의 API 토큰.", "tokenHelpPapra": "Papra: documents:read와 tags:read 권한이 있는 API 키.", "save": "저장하고 테스트", @@ -6723,7 +6723,10 @@ "testOk": "보관함이 {latency}ms 만에 응답했습니다.", "disconnect": "연결 해제", "disconnectTitle": "{name} 연결을 해제할까요?", - "disconnectDescription": "HealthLog가 주소와 토큰을 잊습니다. 이미 가져온 문서는 HealthLog에 남습니다." + "disconnectDescription": "HealthLog가 주소와 토큰을 잊습니다. 이미 가져온 문서는 HealthLog에 남습니다.", + "unavailableExplainer": "이 서버는 더 이상 문서 보관함을 허용하지 않아 아래 연결을 사용할 수 없습니다. 연결은 여전히 삭제할 수 있습니다.", + "moduleOffExplainer": "문서 기능이 꺼져 있어 아래 연결은 사용되지 않습니다. 연결은 여전히 삭제할 수 있습니다.", + "tokenNewAddress": "새 주소에는 API 토큰을 다시 입력해야 합니다." } }, "admin": { @@ -10223,7 +10226,7 @@ "browserOnly": "문서 보관함은 브라우저에서만 사용할 수 있습니다.", "invalidAddress": "https:// 또는 http://로 시작하는 전체 주소를 입력하세요.", "organizationRequired": "Papra 조직 ID를 입력하세요.", - "tokenRequired": "API 토큰을 입력하세요.", + "tokenRequired": "API 토큰을 입력하세요. 새 주소에는 다시 필요합니다.", "fileTooLarge": "파일이 HealthLog가 허용하는 크기보다 큽니다.", "quotaExceeded": "문서 저장 공간이 가득 찼습니다.", "fileType": "HealthLog는 이 파일 형식을 받지 않습니다.", @@ -10231,7 +10234,11 @@ "uploadBusy": "다른 업로드가 아직 진행 중입니다. 잠시 후 다시 시도하세요.", "moduleDisabled": "설정에서 문서 기능이 꺼져 있습니다.", "generic": "문제가 발생했습니다. 다시 시도하세요." - } + }, + "pickMore": "더 고르기", + "finishedHint": "다시 검색하거나 필터를 바꿔 문서를 더 고르세요.", + "maxReached": "한 번에 최대 {max}개 문서를 가져올 수 있습니다.", + "noMatchHint": "이름의 일부로 검색하거나 태그와 날짜를 지워 보세요." } }, "selfDescription": { diff --git a/messages/pl.json b/messages/pl.json index 680921471..7cb49c5ae 100644 --- a/messages/pl.json +++ b/messages/pl.json @@ -6713,7 +6713,7 @@ "organizationId": "Identyfikator organizacji", "organizationHelp": "Część po /organizations/ w pasku adresu Papra.", "token": "Token API", - "tokenSaved": "Zapisano. Zostaw puste, aby go zachować.", + "tokenSaved": "Zapisano dla tego adresu. Zostaw puste, aby go zachować.", "tokenHelpPaperless": "W Paperless-ngx: Mój profil, token API.", "tokenHelpPapra": "W Papra: klucz API z uprawnieniami documents:read i tags:read.", "save": "Zapisz i sprawdź", @@ -6723,7 +6723,10 @@ "testOk": "Archiwum odpowiedziało w {latency} ms.", "disconnect": "Rozłącz", "disconnectTitle": "Rozłączyć {name}?", - "disconnectDescription": "HealthLog zapomina adres i token. Zaimportowane już dokumenty zostają w HealthLog." + "disconnectDescription": "HealthLog zapomina adres i token. Zaimportowane już dokumenty zostają w HealthLog.", + "unavailableExplainer": "Ten serwer nie pozwala już na archiwa dokumentów, więc poniższego połączenia nie da się użyć. Nadal możesz je usunąć.", + "moduleOffExplainer": "Dokumenty są wyłączone, więc poniższe połączenie nie jest używane. Nadal możesz je usunąć.", + "tokenNewAddress": "Nowy adres wymaga ponownego podania tokenu API." } }, "admin": { @@ -10223,7 +10226,7 @@ "browserOnly": "Z archiwów dokumentów można korzystać tylko w przeglądarce.", "invalidAddress": "Wpisz pełny adres, zaczynający się od https:// lub http://.", "organizationRequired": "Wpisz identyfikator organizacji Papra.", - "tokenRequired": "Wpisz token API.", + "tokenRequired": "Wpisz token API. Nowy adres wymaga go ponownie.", "fileTooLarge": "Plik jest większy, niż przyjmuje HealthLog.", "quotaExceeded": "Miejsce na dokumenty jest pełne.", "fileType": "HealthLog nie przyjmuje tego typu pliku.", @@ -10231,7 +10234,11 @@ "uploadBusy": "Trwają jeszcze inne przesyłania. Spróbuj za chwilę.", "moduleDisabled": "Dokumenty są wyłączone w ustawieniach.", "generic": "Coś poszło nie tak. Spróbuj ponownie." - } + }, + "pickMore": "Wybierz więcej", + "finishedHint": "Wyszukaj ponownie lub zmień filtr, aby wybrać więcej dokumentów.", + "maxReached": "Możesz zaimportować naraz najwyżej {max} dokumentów.", + "noMatchHint": "Spróbuj części nazwy albo wyczyść tag i daty." } }, "selfDescription": { diff --git a/src/components/documents/sources/__tests__/document-sources-ui.test.tsx b/src/components/documents/sources/__tests__/document-sources-ui.test.tsx index 2e3f93881..208163bcb 100644 --- a/src/components/documents/sources/__tests__/document-sources-ui.test.tsx +++ b/src/components/documents/sources/__tests__/document-sources-ui.test.tsx @@ -50,6 +50,7 @@ import { queryKeys } from "@/lib/query-keys"; import { ImportFromSourceButton } from "../import-from-source-button"; import { mergeIds } from "../merge-ids"; +import { nextChoiceIndex } from "../segmented-choice"; import { errorCodeOf, sourceErrorMessage } from "../source-errors"; const PAPERLESS = { @@ -162,13 +163,44 @@ describe("", () => { expect(html).toContain( "Allowed on this server: http://paperless.lan:8000, https://papra.example.com", ); - expect(html).toContain("Saved. Leave empty to keep it."); + expect(html).toContain("Saved for this address. Leave empty to keep it."); expect(html).toContain('value="http://paperless.lan:8000"'); // The password field is empty: the server never sends the token back. expect(html).toMatch(/id="document-source-token-paperless"[^>]*value=""/); expect(html).toContain("Disconnect"); }); + it("keeps a stored connection visible and removable once the operator list is gone", () => { + const html = render(, { + available: false, + allowedOrigins: [], + connections: [{ ...PAPERLESS, originAllowed: false }], + }); + expect(html).toContain('data-testid="document-sources-card"'); + expect(html).toContain("no longer allows document archives"); + expect(html).toContain("Disconnect"); + expect(html).not.toContain("Save and test"); + expect(html).not.toContain('id="document-source-token-paperless"'); + }); + + it("keeps a stored connection removable with the documents module off", () => { + modules = { inboundDocuments: false }; + try { + const html = render(, on([PAPERLESS])); + expect(html).toContain("Documents are switched off"); + expect(html).toContain("Disconnect"); + expect(html).not.toContain("Save and test"); + } finally { + modules = { inboundDocuments: true }; + } + }); + + it("uses one tab stop for the system switch", () => { + const html = render(, on([PAPERLESS])); + expect(html.match(/role="radio"[^>]*tabindex="0"/gi)?.length).toBe(1); + expect(html.match(/role="radio"[^>]*tabindex="-1"/gi)?.length).toBe(1); + }); + it("offers no disconnect or test for a system that is not connected", () => { const html = render(, on([])); expect(html).toContain("Not connected"); @@ -223,6 +255,19 @@ describe("source error sentences", () => { }); }); +describe("nextChoiceIndex", () => { + it("moves with the arrows, wraps, and jumps with Home and End", () => { + expect(nextChoiceIndex("ArrowRight", 0, 2)).toBe(1); + expect(nextChoiceIndex("ArrowRight", 1, 2)).toBe(0); + expect(nextChoiceIndex("ArrowLeft", 0, 2)).toBe(1); + expect(nextChoiceIndex("ArrowDown", 0, 3)).toBe(1); + expect(nextChoiceIndex("ArrowUp", 0, 3)).toBe(2); + expect(nextChoiceIndex("Home", 2, 3)).toBe(0); + expect(nextChoiceIndex("End", 0, 3)).toBe(2); + expect(nextChoiceIndex("Enter", 0, 3)).toBeNull(); + }); +}); + describe("mergeIds", () => { it("adds imported ids to a form's selection once, in order", () => { expect(mergeIds(["a", "b"], ["b", "c", "c"])).toEqual(["a", "b", "c"]); diff --git a/src/components/documents/sources/document-source-picker.tsx b/src/components/documents/sources/document-source-picker.tsx index ba9dca00d..09dc62404 100644 --- a/src/components/documents/sources/document-source-picker.tsx +++ b/src/components/documents/sources/document-source-picker.tsx @@ -38,12 +38,14 @@ import { CircleCheck, Loader2, Search, + SearchX, X, } from "lucide-react"; import { useEffect, useMemo, useRef, useState } from "react"; import { Badge } from "@/components/ui/badge"; import { Button } from "@/components/ui/button"; +import { EmptyState } from "@/components/ui/empty-state"; import { DateField } from "@/components/ui/date-field"; import { Input } from "@/components/ui/input"; import { Label } from "@/components/ui/label"; @@ -78,6 +80,7 @@ import { isStoppingError, sourceErrorMessage, } from "./source-errors"; +import { SegmentedChoice } from "./segmented-choice"; import { systemName } from "./use-document-sources"; export interface DocumentPickerLink { @@ -134,6 +137,20 @@ export function DocumentSourcePicker({ const [running, setRunning] = useState(false); const [finished, setFinished] = useState(false); const [stopMessage, setStopMessage] = useState(null); + const [atMax, setAtMax] = useState(false); + /** The latest per-row result, for screen readers (aria-live below). */ + const [announcement, setAnnouncement] = useState(""); + + // A new search or filter starts over: a selection made in one result list + // must not ride along unseen into another, and a finished run's result + // lines belong to the list they were made in. + const startOver = () => { + setSelected(new Map()); + setRuns(new Map()); + setFinished(false); + setStopMessage(null); + setAtMax(false); + }; const debounceRef = useRef | null>(null); useEffect( @@ -145,11 +162,20 @@ export function DocumentSourcePicker({ const onSearchChange = (value: string) => { setSearchDraft(value); if (debounceRef.current) clearTimeout(debounceRef.current); - debounceRef.current = setTimeout( - () => setQ(value.trim()), - SEARCH_DEBOUNCE_MS, - ); + debounceRef.current = setTimeout(() => { + const next = value.trim(); + if (next !== q) { + startOver(); + setQ(next); + } + }, SEARCH_DEBOUNCE_MS); }; + const filterSetter = + (set: (value: string) => void) => + (value: string): void => { + startOver(); + set(value); + }; const reset = () => { setSearchDraft(""); @@ -157,10 +183,7 @@ export function DocumentSourcePicker({ setTag(""); setFrom(""); setTo(""); - setSelected(new Map()); - setRuns(new Map()); - setFinished(false); - setStopMessage(null); + startOver(); }; const switchSystem = (next: DocumentPickerSystem) => { @@ -225,14 +248,18 @@ export function DocumentSourcePicker({ const toggle = (row: DocumentSourceResultDto) => { if (running || finished || !selectable(row)) return; - setSelected((prev) => { - const next = new Map(prev); - if (next.has(row.sourceId)) next.delete(row.sourceId); - else if (next.size < DOCUMENT_PICKER_MAX_SELECTION) { - next.set(row.sourceId, row.title); - } - return next; - }); + const next = new Map(selected); + if (next.has(row.sourceId)) { + next.delete(row.sourceId); + setAtMax(false); + } else if (next.size < DOCUMENT_PICKER_MAX_SELECTION) { + next.set(row.sourceId, row.title); + } else { + // Said rather than silently ignored: the tick that did not appear. + setAtMax(true); + return; + } + setSelected(next); }; async function runImport() { @@ -242,8 +269,12 @@ export function DocumentSourcePicker({ const order = [...selected.keys()]; setRuns(new Map(order.map((id) => [id, { state: "waiting" }]))); const handed: string[] = []; - const mark = (id: string, run: RowRun) => + const mark = (id: string, run: RowRun) => { setRuns((prev) => new Map(prev).set(id, run)); + const title = selected.get(id) ?? id; + const said = runLabel(t, run); + if (said) setAnnouncement(`${title}: ${said}`); + }; for (const sourceId of order) { mark(sourceId, { state: "working" }); @@ -318,14 +349,26 @@ export function DocumentSourcePicker({ })}

{finished ? ( - +
+ + +
) : ( - ))} -
+ {systems.length > 1 && active ? ( + ) : null}
@@ -423,7 +449,7 @@ export function DocumentSourcePicker({ setTag(e.target.value)} + onChange={(e) => filterSetter(setTag)(e.target.value)} disabled={tags.isPending && !tags.isError} > @@ -441,7 +467,7 @@ export function DocumentSourcePicker({
@@ -452,7 +478,7 @@ export function DocumentSourcePicker({
@@ -461,6 +487,7 @@ export function DocumentSourcePicker({ + ); + })} +
+ ); +} diff --git a/src/components/documents/sources/use-document-sources.ts b/src/components/documents/sources/use-document-sources.ts index 13b1a4692..e19329ca2 100644 --- a/src/components/documents/sources/use-document-sources.ts +++ b/src/components/documents/sources/use-document-sources.ts @@ -3,12 +3,12 @@ /** * The document picker's availability for the person on this screen (#1038). * - * Reads `GET /api/documents/sources` only when it can matter: the documents - * module is on and the person is in their own record with the right to manage - * documents. A delegate or guardian never asks (the route would refuse them - * anyway), so the import buttons and the settings card render nothing for - * them. `connected` is the list of systems with a usable connection, in a - * stable order; an entry whose origin the operator removed is left out. + * Reads `GET /api/documents/sources` only in the person's own record: a + * delegate or guardian never asks (the route would refuse them anyway), so + * the import buttons and the settings card render nothing for them. + * `connected` is the list of systems with a usable connection (the documents + * module on, the operator's list set, the origin still on it), in a stable + * order; the import buttons read only that. */ import { useQuery } from "@tanstack/react-query"; @@ -24,25 +24,32 @@ import { queryKeys } from "@/lib/query-keys"; export function useDocumentSourcesStatus() { const { user } = useAuth(); const { canManageDomain, inSharedRecord } = useRecordCapabilities(); - const eligible = - user?.modules?.inboundDocuments === true && + // The status read touches only the person's own rows and never the + // network, so it runs in one's own record whatever the module says: a + // connection stored before Documents was switched off, or before the + // operator removed the list, must stay visible so it can be deleted. + const ownRecord = + user !== null && + user !== undefined && !inSharedRecord && canManageDomain("documents"); + const eligible = ownRecord && user?.modules?.inboundDocuments === true; const query = useQuery({ queryKey: queryKeys.documentSourcesStatus(), - enabled: eligible, + enabled: ownRecord, queryFn: () => apiGet("/api/documents/sources"), staleTime: 60_000, }); - const connected: DocumentPickerSystem[] = eligible - ? (query.data?.connections ?? []) - .filter((c) => c.originAllowed) - .map((c) => c.system) - : []; + const connected: DocumentPickerSystem[] = + eligible && query.data?.available + ? query.data.connections + .filter((c) => c.originAllowed) + .map((c) => c.system) + : []; - return { eligible, query, connected }; + return { ownRecord, eligible, query, connected }; } /** Display name of a system; product names, not translated. */ diff --git a/src/components/settings/integrations/document-sources-card.tsx b/src/components/settings/integrations/document-sources-card.tsx index d63cfd30d..8a0cf216e 100644 --- a/src/components/settings/integrations/document-sources-card.tsx +++ b/src/components/settings/integrations/document-sources-card.tsx @@ -39,6 +39,7 @@ import { SettingsCardActions } from "@/components/settings/_card-actions"; import { SettingsCardHeader } from "@/components/settings/_card-header"; import { SettingsCard } from "@/components/settings/settings-card"; import { WrittenOutcomeLine } from "@/components/outcome/written-outcome-line"; +import { SegmentedChoice } from "@/components/documents/sources/segmented-choice"; import { errorCodeOf, sourceErrorMessage, @@ -56,19 +57,28 @@ import { } from "@/lib/documents/sources/types"; import { useFormatters, useTranslations } from "@/lib/i18n/context"; import { invalidateKeys, queryKeys } from "@/lib/query-keys"; -import { cn } from "@/lib/utils"; type Outcome = { tone: "success" | "error"; message: string } | null; export function DocumentSourcesCard() { const { t } = useTranslations(); - const { eligible, query } = useDocumentSourcesStatus(); - const [system, setSystem] = useState("PAPERLESS"); + const { ownRecord, eligible, query } = useDocumentSourcesStatus(); + const [picked, setPicked] = useState("PAPERLESS"); - if (!eligible || !query.data?.available) return null; + const data = query.data; + const connections = data?.connections ?? []; + // Editing needs the operator's list and the documents module. Without + // either, a connection that is still stored stays on the card so its token + // can be deleted; with none stored there is nothing to show. + const editable = eligible && data?.available === true; + if (!ownRecord || !data || (!editable && connections.length === 0)) { + return null; + } - const connections = query.data.connections; - const connectedCount = connections.length; + const options: readonly DocumentPickerSystem[] = editable + ? DOCUMENT_PICKER_SYSTEMS + : connections.map((c) => c.system); + const system = options.includes(picked) ? picked : options[0]; return ( @@ -77,7 +87,7 @@ export function DocumentSourcesCard() { title={t("settings.documentSources.title")} description={t("settings.documentSources.description")} status={ - connectedCount > 0 ? ( + connections.length > 0 ? ( {t("settings.documentSources.statusConnected")} @@ -85,62 +95,67 @@ export function DocumentSourcesCard() { } /> -

{t("settings.documentSources.explainer")}

+

+ {editable + ? t("settings.documentSources.explainer") + : eligible + ? t("settings.documentSources.unavailableExplainer") + : t("settings.documentSources.moduleOffExplainer")} +

-
- {DOCUMENT_PICKER_SYSTEMS.map((option) => { - const isConnected = connections.some((c) => c.system === option); - return ( - - ); - })} -
+ + )} + /> + ) : null} c.system === system) ?? null} - allowedOrigins={query.data.allowedOrigins} + allowedOrigins={data.allowedOrigins} + editable={editable} /> ); } +/** The origin of an address, or null while it is not one yet. */ +function originOf(value: string): string | null { + try { + return new URL(value.trim()).origin; + } catch { + return null; + } +} + function SourceForm({ system, connection, allowedOrigins, + editable, }: { system: DocumentPickerSystem; connection: DocumentSourceConnectionDto | null; allowedOrigins: string[]; + /** False when only removing the stored connection is possible. */ + editable: boolean; }) { const { t } = useTranslations(); const format = useFormatters(); @@ -225,10 +240,17 @@ function SourceForm({ }); const busy = save.isPending || test.isPending || disconnect.isPending; + // The stored token is reused only on the address's own origin; the server + // refuses to send it anywhere else, so the form asks for it again. + const keepsToken = + connection !== null && + originOf(baseUrl) !== null && + originOf(baseUrl) === originOf(connection.baseUrl); const canSave = + editable && baseUrl.trim() !== "" && (!isPapra || organizationId.trim() !== "") && - (connection !== null || token.trim() !== "") && + (keepsToken || token.trim() !== "") && dirty; return ( @@ -253,7 +275,7 @@ function SourceForm({ : t("settings.documentSources.statusNotConnected")}

- {connection && !connection.originAllowed ? ( + {editable && connection && !connection.originAllowed ? (

) : null} -

- - setBaseUrl(e.target.value)} - placeholder={allowedOrigins[0] ?? "https://"} - maxLength={2048} - autoComplete="off" - inputMode="url" - spellCheck={false} - autoCapitalize="none" - /> -

- {t("settings.documentSources.allowedOrigins", { - origins: allowedOrigins.join(", "), - })} -

-
+ {editable ? ( + <> +
+ + setBaseUrl(e.target.value)} + placeholder={allowedOrigins[0] ?? "https://"} + maxLength={2048} + autoComplete="off" + inputMode="url" + spellCheck={false} + autoCapitalize="none" + /> +

+ {t("settings.documentSources.allowedOrigins", { + origins: allowedOrigins.join(", "), + })} +

+
- {isPapra ? ( -
- - setOrganizationId(e.target.value)} - maxLength={128} - autoComplete="off" - spellCheck={false} - autoCapitalize="none" - /> -

- {t("settings.documentSources.organizationHelp")} -

-
+ {isPapra ? ( +
+ + setOrganizationId(e.target.value)} + maxLength={128} + autoComplete="off" + spellCheck={false} + autoCapitalize="none" + /> +

+ {t("settings.documentSources.organizationHelp")} +

+
+ ) : null} + +
+ + setToken(e.target.value)} + placeholder={ + keepsToken + ? t("settings.documentSources.tokenSaved") + : undefined + } + maxLength={512} + autoComplete="off" + spellCheck={false} + autoCapitalize="none" + /> +

+ {isPapra + ? t("settings.documentSources.tokenHelpPapra") + : t("settings.documentSources.tokenHelpPaperless")} +

+ {connection && !keepsToken && baseUrl.trim() !== "" ? ( +

+ {t("settings.documentSources.tokenNewAddress")} +

+ ) : null} +
+ + ) : connection ? ( +

+ {connection.baseUrl} +

) : null} -
- - setToken(e.target.value)} - placeholder={ - connection ? t("settings.documentSources.tokenSaved") : undefined - } - maxLength={512} - autoComplete="off" - spellCheck={false} - autoCapitalize="none" - /> -

- {isPapra - ? t("settings.documentSources.tokenHelpPapra") - : t("settings.documentSources.tokenHelpPaperless")} -

-
- {outcome ? ( ) : null} - {connection ? ( + {connection && editable ? ( + {editable ? ( + + ) : null} ); From 933553a7bda586fc460de294348bf1c05bc9690c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Sat, 26 Sep 2026 14:33:23 +0200 Subject: [PATCH 13/18] docs(documents): source instance in the workflow recipe and the key rules; token and removal rules for the picker --- docs/integrations/document-import.md | 22 ++++++++++++++++------ docs/self-hosting/document-sources.md | 12 ++++++++++-- 2 files changed, 26 insertions(+), 8 deletions(-) diff --git a/docs/integrations/document-import.md b/docs/integrations/document-import.md index 48e7d39c1..0c5c54628 100644 --- a/docs/integrations/document-import.md +++ b/docs/integrations/document-import.md @@ -43,12 +43,13 @@ This needs Paperless-ngx 3.0 or later, the first release with the - **Use parameters for webhook body:** on. **Send webhook payload as JSON:** off. **Webhook params:** - | Name | Value | - | -------------- | ---------------------------------------------------- | - | `title` | `{{doc_title}}` | - | `documentDate` | `{{created_year}}-{{created_month}}-{{created_day}}` | - | `sourceSystem` | `PAPERLESS` | - | `sourceId` | `{{doc_id}}` | + | Name | Value | + | ---------------- | -------------------------------------------------------- | + | `title` | `{{doc_title}}` | + | `documentDate` | `{{created_year}}-{{created_month}}-{{created_day}}` | + | `sourceSystem` | `PAPERLESS` | + | `sourceId` | `{{doc_id}}` | + | `sourceInstance` | `https://paperless.example.com` (your Paperless address) | - **Include document:** on. Paperless then sends the original file along with the parameters, which is exactly what HealthLog expects. @@ -231,6 +232,15 @@ per account: if you push documents from two other systems, make sure their ids cannot collide, for example by prefixing them (`nextcloud-123`, `scanner-123`). +Since v1.39.3 a key can also name its instance: `sourceInstance`, the +address of the Paperless-ngx or Papra it came from (only its origin, +`scheme://host[:port]`, counts). With it, document 123 on one Paperless and +document 123 on another are two documents. The import script sends it +automatically; add it to a Paperless workflow as shown above. A key sent or +stored without an instance (everything imported with v1.39.2, or a workflow +without the parameter) matches that id on any instance of its system, so +nothing imported earlier is stored twice or brought back after a deletion. + The same file sent under a different id is stored once, and HealthLog remembers the extra id for it, up to 20 extra ids per document. A 21st is refused with `409` (`documents.inbound.sourceAliasLimit`) and nothing is diff --git a/docs/self-hosting/document-sources.md b/docs/self-hosting/document-sources.md index 96588771b..8b35367a0 100644 --- a/docs/self-hosting/document-sources.md +++ b/docs/self-hosting/document-sources.md @@ -51,7 +51,9 @@ Settings → Integrations → Document archives: - **Papra:** the base address, the organization id (from the Papra address bar) and an API key with `documents:read` and `tags:read`. -HealthLog tests the connection before saving it. The token is stored +HealthLog tests the connection before saving it. A saved token is only ever +sent to the address it was saved for: changing the address to another origin +asks for the token again. The token is stored encrypted (AES-256-GCM, rotated with the other keys by `scripts/rotate-encryption-key.ts`) and is never shown again, not even in part. @@ -70,7 +72,10 @@ limit for a document). - Imports count against the person's normal upload allowance (60 an hour) and storage quota. - A document that is already in HealthLog is not stored twice, and one the - person deleted in HealthLog is not brought back by picking it again. These + person deleted in HealthLog is not brought back by picking it again. Each + import is keyed by system, instance (the archive's origin) and document id, + so the same id on two Paperless-ngx instances is two documents; a key + imported before v1.39.3 carries no instance and matches any instance. These are the same rules the document token and `scripts/import-documents.mjs` follow. - A picked document follows the person's own automatic AI reading setting. @@ -80,3 +85,6 @@ limit for a document). each person reconnects (one form); the imported documents themselves are in the backup with the rest of the vault. - Delete all data and account deletion remove the connection and its token. +- A stored connection stays visible under Settings, read-only, when this + variable is unset or the person switched Documents off, so it can still be + removed. From c5a63934bd44e702f13a13efe6f3f15b7afcc948 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Sat, 26 Sep 2026 14:42:13 +0200 Subject: [PATCH 14/18] fix(documents): an import of bytes deleted in HealthLog stays deleted under any source key Keyed uploads and picker imports also check deleted, not yet purged documents by content and answer deleted, remembering the new key on the tombstone so the purge carries it into the ledger. A hand upload without a key still stores a new copy. No content hash of a deleted document is kept past the purge. --- src/lib/documents/ingest.ts | 32 +++++++++++ tests/integration/document-sources.test.ts | 64 ++++++++++++++++++++++ 2 files changed, 96 insertions(+) diff --git a/src/lib/documents/ingest.ts b/src/lib/documents/ingest.ts index 2e21291c4..0932ba813 100644 --- a/src/lib/documents/ingest.ts +++ b/src/lib/documents/ingest.ts @@ -16,6 +16,8 @@ * id names nothing the caller owns; * - sha256 duplicate detection against live rows, remembering the source key * for the existing document so a later delete keeps that key deleted too; + * for an import, also against deleted (not yet purged) rows, which answer + * "deleted" rather than storing the file again; * - the quota gate and the insert in one transaction under a per-user * advisory lock, and the race on either partial unique index resolved the * way the fast paths above would have answered; @@ -216,6 +218,36 @@ export async function ingestDocument( }); if (existing) return answerDuplicate(input, existing); + // An import (a source key is present) of bytes the person deleted here, + // arriving under a key HealthLog has not seen: another system, another + // instance, or a re-scan with a new id. The deletion was a decision about + // the document, not about one key, so it is answered "deleted", and the + // new key is remembered on the tombstone so the purge carries it into the + // ledger. After the 30-day purge the bytes and their hash are gone and only + // the keys remain; no content hash of a deleted document is kept. A person + // uploading the file by hand (no key) is making a new copy on purpose. + if (sourceSystem && sourceId) { + const tombstone = await prisma.inboundDocument.findFirst({ + where: { userId, contentSha256, deletedAt: { not: null } }, + select: { id: true }, + orderBy: { deletedAt: "desc" }, + }); + if (tombstone) { + const remembered = await rememberSourceAlias( + userId, + tombstone.id, + sourceSystem, + sourceId, + sourceInstance, + ); + if (remembered === "limit") return { kind: "aliasLimit" }; + return { + kind: "sourceKey", + match: { state: "deleted", id: tombstone.id }, + }; + } + } + // Quota gate + insert + pre-links in ONE transaction. Usage counts every // non-purged row — tombstones still hold TOAST bytes, so "deleted" bytes // are never invisible weight (undo-delete never changes usage). diff --git a/tests/integration/document-sources.test.ts b/tests/integration/document-sources.test.ts index 0bcb842e6..70776ef58 100644 --- a/tests/integration/document-sources.test.ts +++ b/tests/integration/document-sources.test.ts @@ -638,6 +638,70 @@ describe("search and import", () => { }); describe("review follow-ups", () => { + it("keeps a document deleted in HealthLog deleted when the same bytes arrive under another source key", async () => { + await seedOwner(); + await connect(); + const prisma = getPrismaClient(); + const first = await importDoc({ sourceId: "41" }); + const id = (await first.json()).data.documentId as string; + await prisma.inboundDocument.update({ + where: { id }, + data: { deletedAt: new Date() }, + }); + + // The same file, uploaded by another importer under its own key (a + // document token or the script sending from another system). + const form = new FormData(); + form.append( + "file", + new Blob([ + new Uint8Array( + Buffer.concat([DOCS[0].bytes, Buffer.from(`% ${ORIGIN}\n`)]), + ), + ]), + "x.pdf", + ); + form.append("sourceSystem", "PAPRA"); + form.append("sourceId", "doc_same_bytes"); + const { POST } = await import("@/app/api/documents/inbound/route"); + const res = await (POST as unknown as (r: Request) => Promise)( + new Request("http://localhost/api/documents/inbound", { + method: "POST", + body: form, + }), + ); + expect(res.status).toBe(200); + expect((await res.json()).data).toMatchObject({ deleted: true }); + expect(await prisma.inboundDocument.count()).toBe(1); + // The new key is remembered on the deleted document, so the purge + // carries it into the ledger. + expect( + await prisma.documentSourceAlias.count({ + where: { documentId: id, sourceId: "doc_same_bytes" }, + }), + ).toBe(1); + + // A person uploading the same file by hand, without a key, is a + // deliberate new copy and is stored. + const manual = new FormData(); + manual.append( + "file", + new Blob([ + new Uint8Array( + Buffer.concat([DOCS[0].bytes, Buffer.from(`% ${ORIGIN}\n`)]), + ), + ]), + "x.pdf", + ); + const byHand = await (POST as unknown as (r: Request) => Promise)( + new Request("http://localhost/api/documents/inbound", { + method: "POST", + body: manual, + }), + ); + expect(byHand.status).toBe(201); + }); + it("never sends the stored token to a new origin", async () => { await seedOwner(); await connect(); From 057281998f8ddb42495cd9c7dfdc69e769d6f2fa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Sat, 26 Sep 2026 14:46:01 +0200 Subject: [PATCH 15/18] fix(documents): read Papra as it answers Papra organization ids are checked against org_ plus 24 characters before anything is sent. A 403 user.not_in_organization and a 400 server.invalid_request.params read as no such organization instead of a refused key; the error body is read for its code and never passed on. The connection test also reads the tags, so a key without tags:read fails at save (permissionMissing) instead of on the first filter. Name searches quote each word, so a leading dash is not a negation, and the date range goes to Papra as date:>= / date:<= with the per-row check kept as a net. --- docs/api/openapi.yaml | 30 ++++--- messages/de.json | 7 +- messages/en.json | 7 +- messages/es.json | 7 +- messages/fr.json | 7 +- messages/it.json | 7 +- messages/ko.json | 7 +- messages/pl.json | 7 +- .../documents/sources/source-errors.ts | 3 + .../sources/__tests__/clients.test.ts | 90 +++++++++++++++++-- src/lib/documents/sources/clients.ts | 52 ++++++++++- src/lib/documents/sources/http.ts | 60 ++++++++++++- src/lib/documents/sources/route-support.ts | 3 + src/lib/documents/sources/types.ts | 5 +- src/lib/openapi/error-codes.ts | 1 + src/lib/openapi/routes/document-sources.ts | 4 +- src/lib/validations/document-sources.ts | 4 +- 17 files changed, 245 insertions(+), 56 deletions(-) diff --git a/docs/api/openapi.yaml b/docs/api/openapi.yaml index 5be68b466..22c56b089 100644 --- a/docs/api/openapi.yaml +++ b/docs/api/openapi.yaml @@ -21065,7 +21065,7 @@ paths: maxLength: 2048 organizationId: type: string - pattern: ^[A-Za-z0-9_-]{1,128}$ + pattern: ^org_[a-z0-9]{24}$ token: type: string minLength: 1 @@ -21085,7 +21085,9 @@ paths: "404": &a55 description: "`documents.sources.unavailable`: the operator has not listed any origin in `DOCUMENT_SOURCE_ORIGINS`, so the picker is off. `documents.sources.notConnected`: no connection is saved for this system. - `documents.sources.notFound`: the source has no such document, or (Papra) no such organization." + `documents.sources.notFound`: the source has no such document, or (Papra) no such organization, including + one the key's user is not a member of (Papra 403 `user.not_in_organization`) and one Papra cannot parse + (400)." content: application/json: schema: @@ -21104,7 +21106,8 @@ paths: all, its HTTP status in `meta.upstreamStatus` (its body is never passed on): `documents.sources.unreachable` (no connection, timeout, or a destination refused at dial time), `documents.sources.redirected` (the source answered with a redirect, which is never followed), `documents.sources.authRefused` (401/403 from the - source), `documents.sources.versionTooOld` (Paperless-ngx older than 2.16 / API version 9), + source), `documents.sources.permissionMissing` (Papra: the key cannot read tags; needs `tags:read`), + `documents.sources.versionTooOld` (Paperless-ngx older than 2.16 / API version 9), `documents.sources.badResponse` (not the documented JSON, or larger than 1 MiB)." content: application/json: @@ -21327,9 +21330,10 @@ paths: "404": description: "`documents.sources.unavailable`: the operator has not listed any origin in `DOCUMENT_SOURCE_ORIGINS`, so the picker is off. `documents.sources.notConnected`: no connection is saved for this system. - `documents.sources.notFound`: the source has no such document, or (Papra) no such organization. - `documents.sources.linkTargetNotFound`: the record in `link` is not the caller's own live conditionEpisode / - encounter / vaccination." + `documents.sources.notFound`: the source has no such document, or (Papra) no such organization, including + one the key's user is not a member of (Papra 403 `user.not_in_organization`) and one Papra cannot parse + (400). `documents.sources.linkTargetNotFound`: the record in `link` is not the caller's own live + conditionEpisode / encounter / vaccination." content: application/json: schema: @@ -31239,13 +31243,13 @@ components: documents.sources.authRefused, documents.sources.badResponse, documents.sources.browserOnly, documents.sources.invalidAddress, documents.sources.linkTargetNotFound, documents.sources.notConnected, documents.sources.notFound, documents.sources.organizationRequired, documents.sources.originNotAllowed, - documents.sources.rateLimited, documents.sources.redirected, documents.sources.tokenRequired, - documents.sources.unavailable, documents.sources.unreachable, documents.sources.versionTooOld; encounter - — encounter.invalid, encounter.link.invalid, encounter.practitioner-not-found, - encounter.reminder-conflict, encounter.reminder-not-found; environment — environment.invalid, - environment.no_home, environment.range_too_large, environment.travel.not_found; export — - export.selection.empty, export.selection.unknown_leaf; familyHistory — familyHistory.invalid; feedback — - feedback.body.invalid; health_score_config — health_score_config.invalid, + documents.sources.permissionMissing, documents.sources.rateLimited, documents.sources.redirected, + documents.sources.tokenRequired, documents.sources.unavailable, documents.sources.unreachable, + documents.sources.versionTooOld; encounter — encounter.invalid, encounter.link.invalid, + encounter.practitioner-not-found, encounter.reminder-conflict, encounter.reminder-not-found; environment + — environment.invalid, environment.no_home, environment.range_too_large, environment.travel.not_found; + export — export.selection.empty, export.selection.unknown_leaf; familyHistory — familyHistory.invalid; + feedback — feedback.body.invalid; health_score_config — health_score_config.invalid, health_score_config.too_narrow; illness — illness.day-log.invalid, illness.disabled, illness.episode.chronic-no-resolve, illness.episode.invalid, illness.episode.invalid-window, illness.episode.parent-not-found, illness.insights.invalid; import — import.write_failed; ingest — diff --git a/messages/de.json b/messages/de.json index b0426eec2..3dead10f6 100644 --- a/messages/de.json +++ b/messages/de.json @@ -6711,7 +6711,7 @@ "baseUrl": "Adresse", "allowedOrigins": "Auf diesem Server erlaubt: {origins}", "organizationId": "Organisations-ID", - "organizationHelp": "Der Teil nach /organizations/ in der Adresszeile von Papra.", + "organizationHelp": "Der Teil nach /organizations/ in der Adresszeile von Papra. Papra-Organisations-IDs sehen aus wie org_ gefolgt von 24 Buchstaben und Ziffern.", "token": "API-Token", "tokenSaved": "Für diese Adresse gespeichert. Leer lassen, um es zu behalten.", "tokenHelpPaperless": "In Paperless-ngx: Mein Profil, API-Token.", @@ -10220,7 +10220,7 @@ "authRefused": "Das Archiv hat das Token abgelehnt. Prüfe das Token und seine Berechtigungen.", "versionTooOld": "Das Archiv ist zu alt. Nötig ist Paperless-ngx 2.16 oder neuer.", "badResponse": "Die Antwort des Archivs ließ sich nicht lesen. Prüfe, ob die Adresse auf das Archiv selbst zeigt.", - "notFound": "Das Archiv kennt dieses Dokument oder diese Organisation nicht.", + "notFound": "Das Archiv kennt dieses Dokument nicht, oder die Organisations-ID ist falsch oder nicht deine.", "linkTargetNotFound": "Der Eintrag, mit dem verknüpft werden sollte, existiert nicht mehr.", "rateLimited": "Zu viele Anfragen. Warte eine Minute und versuche es erneut.", "browserOnly": "Dokumentarchive lassen sich nur im Browser verwenden.", @@ -10233,7 +10233,8 @@ "uploadLimit": "Das stündliche Upload-Limit ist erreicht. Versuche es später erneut.", "uploadBusy": "Andere Uploads laufen noch. Versuche es gleich noch einmal.", "moduleDisabled": "Dokumente sind in den Einstellungen ausgeschaltet.", - "generic": "Etwas ist schiefgelaufen. Versuche es erneut." + "generic": "Etwas ist schiefgelaufen. Versuche es erneut.", + "permissionMissing": "Der Papra-Schlüssel darf keine Tags lesen. Gib ihm die Berechtigung tags:read." }, "pickMore": "Weitere auswählen", "finishedHint": "Suche erneut oder ändere einen Filter, um weitere Dokumente auszuwählen.", diff --git a/messages/en.json b/messages/en.json index f42ec1e19..2e3c9feb5 100644 --- a/messages/en.json +++ b/messages/en.json @@ -6711,7 +6711,7 @@ "baseUrl": "Address", "allowedOrigins": "Allowed on this server: {origins}", "organizationId": "Organization id", - "organizationHelp": "The part after /organizations/ in the Papra address bar.", + "organizationHelp": "The part after /organizations/ in the Papra address bar. Papra organization ids look like org_ followed by 24 letters and digits.", "token": "API token", "tokenSaved": "Saved for this address. Leave empty to keep it.", "tokenHelpPaperless": "In Paperless-ngx: My Profile, API token.", @@ -10220,7 +10220,7 @@ "authRefused": "The archive refused the token. Check the token and its permissions.", "versionTooOld": "The archive is too old. Paperless-ngx 2.16 or later is needed.", "badResponse": "The archive's answer could not be read. Check that the address points at the archive itself.", - "notFound": "The archive has no such document, or no such organization.", + "notFound": "The archive has no such document, or the organization id is wrong or not yours.", "linkTargetNotFound": "The record to link to no longer exists.", "rateLimited": "Too many requests. Wait a minute and try again.", "browserOnly": "Document archives can only be used in the browser.", @@ -10233,7 +10233,8 @@ "uploadLimit": "The hourly upload limit is reached. Try again later.", "uploadBusy": "Other uploads are still running. Try again in a moment.", "moduleDisabled": "Documents are switched off in Settings.", - "generic": "Something went wrong. Try again." + "generic": "Something went wrong. Try again.", + "permissionMissing": "The Papra key cannot read tags. Give it the tags:read permission." }, "pickMore": "Pick more", "finishedHint": "Search again or change a filter to pick more documents.", diff --git a/messages/es.json b/messages/es.json index 6f9e5127b..5447d74ba 100644 --- a/messages/es.json +++ b/messages/es.json @@ -6711,7 +6711,7 @@ "baseUrl": "Dirección", "allowedOrigins": "Permitido en este servidor: {origins}", "organizationId": "Id de organización", - "organizationHelp": "La parte que sigue a /organizations/ en la barra de direcciones de Papra.", + "organizationHelp": "La parte que sigue a /organizations/ en la barra de direcciones de Papra. Los id de organización de Papra tienen la forma org_ seguida de 24 letras y cifras.", "token": "Token de la API", "tokenSaved": "Guardado para esta dirección. Déjalo vacío para conservarlo.", "tokenHelpPaperless": "En Paperless-ngx: Mi perfil, token de la API.", @@ -10220,7 +10220,7 @@ "authRefused": "El archivo rechazó el token. Comprueba el token y sus permisos.", "versionTooOld": "El archivo es demasiado antiguo. Se necesita Paperless-ngx 2.16 o posterior.", "badResponse": "No se pudo leer la respuesta del archivo. Comprueba que la dirección apunta al propio archivo.", - "notFound": "El archivo no tiene ese documento o esa organización.", + "notFound": "El archivo no tiene ese documento, o el id de organización es incorrecto o no es tuyo.", "linkTargetNotFound": "El registro que se iba a vincular ya no existe.", "rateLimited": "Demasiadas solicitudes. Espera un minuto y vuelve a intentarlo.", "browserOnly": "Los archivos de documentos solo se pueden usar en el navegador.", @@ -10233,7 +10233,8 @@ "uploadLimit": "Se alcanzó el límite de subidas por hora. Inténtalo más tarde.", "uploadBusy": "Aún hay otras subidas en curso. Inténtalo en un momento.", "moduleDisabled": "Los documentos están desactivados en Ajustes.", - "generic": "Algo salió mal. Inténtalo de nuevo." + "generic": "Algo salió mal. Inténtalo de nuevo.", + "permissionMissing": "La clave de Papra no puede leer etiquetas. Dale el permiso tags:read." }, "pickMore": "Elegir más", "finishedHint": "Busca de nuevo o cambia un filtro para elegir más documentos.", diff --git a/messages/fr.json b/messages/fr.json index 93f029e4c..e75cca526 100644 --- a/messages/fr.json +++ b/messages/fr.json @@ -6711,7 +6711,7 @@ "baseUrl": "Adresse", "allowedOrigins": "Autorisé sur ce serveur : {origins}", "organizationId": "Identifiant d'organisation", - "organizationHelp": "La partie après /organizations/ dans la barre d'adresse de Papra.", + "organizationHelp": "La partie après /organizations/ dans la barre d'adresse de Papra. Les identifiants d'organisation Papra ont la forme org_ suivi de 24 lettres et chiffres.", "token": "Jeton d'API", "tokenSaved": "Enregistré pour cette adresse. Laissez vide pour le conserver.", "tokenHelpPaperless": "Dans Paperless-ngx : Mon profil, jeton d'API.", @@ -10220,7 +10220,7 @@ "authRefused": "L'archive a refusé le jeton. Vérifiez le jeton et ses droits.", "versionTooOld": "L'archive est trop ancienne. Paperless-ngx 2.16 ou plus récent est nécessaire.", "badResponse": "La réponse de l'archive est illisible. Vérifiez que l'adresse pointe vers l'archive elle-même.", - "notFound": "L'archive ne connaît pas ce document ou cette organisation.", + "notFound": "L'archive ne connaît pas ce document, ou l'identifiant d'organisation est faux ou n'est pas le vôtre.", "linkTargetNotFound": "L'élément auquel lier le document n'existe plus.", "rateLimited": "Trop de requêtes. Patientez une minute et réessayez.", "browserOnly": "Les archives de documents ne s'utilisent que dans le navigateur.", @@ -10233,7 +10233,8 @@ "uploadLimit": "La limite horaire d'envois est atteinte. Réessayez plus tard.", "uploadBusy": "D'autres envois sont en cours. Réessayez dans un instant.", "moduleDisabled": "Les documents sont désactivés dans les réglages.", - "generic": "Un problème est survenu. Réessayez." + "generic": "Un problème est survenu. Réessayez.", + "permissionMissing": "La clé Papra ne peut pas lire les étiquettes. Donnez-lui le droit tags:read." }, "pickMore": "En choisir d'autres", "finishedHint": "Cherchez à nouveau ou changez un filtre pour choisir d'autres documents.", diff --git a/messages/it.json b/messages/it.json index 23f268c99..802991bbc 100644 --- a/messages/it.json +++ b/messages/it.json @@ -6711,7 +6711,7 @@ "baseUrl": "Indirizzo", "allowedOrigins": "Consentito su questo server: {origins}", "organizationId": "Id organizzazione", - "organizationHelp": "La parte dopo /organizations/ nella barra degli indirizzi di Papra.", + "organizationHelp": "La parte dopo /organizations/ nella barra degli indirizzi di Papra. Gli id organizzazione di Papra hanno la forma org_ seguita da 24 lettere e cifre.", "token": "Token API", "tokenSaved": "Salvato per questo indirizzo. Lascia vuoto per mantenerlo.", "tokenHelpPaperless": "In Paperless-ngx: Il mio profilo, token API.", @@ -10220,7 +10220,7 @@ "authRefused": "L'archivio ha rifiutato il token. Controlla il token e i suoi permessi.", "versionTooOld": "L'archivio è troppo vecchio. Serve Paperless-ngx 2.16 o successivo.", "badResponse": "Non è stato possibile leggere la risposta dell'archivio. Verifica che l'indirizzo punti all'archivio stesso.", - "notFound": "L'archivio non ha questo documento o questa organizzazione.", + "notFound": "L'archivio non ha questo documento, oppure l'id dell'organizzazione è sbagliato o non è tuo.", "linkTargetNotFound": "L'elemento da collegare non esiste più.", "rateLimited": "Troppe richieste. Attendi un minuto e riprova.", "browserOnly": "Gli archivi di documenti si usano solo nel browser.", @@ -10233,7 +10233,8 @@ "uploadLimit": "È stato raggiunto il limite orario di caricamenti. Riprova più tardi.", "uploadBusy": "Altri caricamenti sono ancora in corso. Riprova tra un momento.", "moduleDisabled": "I documenti sono disattivati nelle impostazioni.", - "generic": "Qualcosa è andato storto. Riprova." + "generic": "Qualcosa è andato storto. Riprova.", + "permissionMissing": "La chiave Papra non può leggere le etichette. Dalle il permesso tags:read." }, "pickMore": "Scegline altri", "finishedHint": "Cerca di nuovo o cambia un filtro per scegliere altri documenti.", diff --git a/messages/ko.json b/messages/ko.json index fef692073..ace3dd8ac 100644 --- a/messages/ko.json +++ b/messages/ko.json @@ -6711,7 +6711,7 @@ "baseUrl": "주소", "allowedOrigins": "이 서버에서 허용됨: {origins}", "organizationId": "조직 ID", - "organizationHelp": "Papra 주소 표시줄에서 /organizations/ 다음 부분입니다.", + "organizationHelp": "Papra 주소 표시줄에서 /organizations/ 다음 부분입니다. Papra 조직 ID는 org_ 뒤에 영문자와 숫자 24자가 오는 형식입니다.", "token": "API 토큰", "tokenSaved": "이 주소에 저장됨. 유지하려면 비워 두세요.", "tokenHelpPaperless": "Paperless-ngx: 내 프로필의 API 토큰.", @@ -10220,7 +10220,7 @@ "authRefused": "보관함이 토큰을 거부했습니다. 토큰과 권한을 확인하세요.", "versionTooOld": "보관함 버전이 너무 오래되었습니다. Paperless-ngx 2.16 이상이 필요합니다.", "badResponse": "보관함의 응답을 읽을 수 없습니다. 주소가 보관함 자체를 가리키는지 확인하세요.", - "notFound": "보관함에 해당 문서나 조직이 없습니다.", + "notFound": "보관함에 해당 문서가 없거나, 조직 ID가 잘못되었거나 본인의 조직이 아닙니다.", "linkTargetNotFound": "연결할 기록이 더 이상 존재하지 않습니다.", "rateLimited": "요청이 너무 많습니다. 1분 후에 다시 시도하세요.", "browserOnly": "문서 보관함은 브라우저에서만 사용할 수 있습니다.", @@ -10233,7 +10233,8 @@ "uploadLimit": "시간당 업로드 한도에 도달했습니다. 나중에 다시 시도하세요.", "uploadBusy": "다른 업로드가 아직 진행 중입니다. 잠시 후 다시 시도하세요.", "moduleDisabled": "설정에서 문서 기능이 꺼져 있습니다.", - "generic": "문제가 발생했습니다. 다시 시도하세요." + "generic": "문제가 발생했습니다. 다시 시도하세요.", + "permissionMissing": "Papra 키가 태그를 읽을 수 없습니다. tags:read 권한을 부여하세요." }, "pickMore": "더 고르기", "finishedHint": "다시 검색하거나 필터를 바꿔 문서를 더 고르세요.", diff --git a/messages/pl.json b/messages/pl.json index 7cb49c5ae..fef668a57 100644 --- a/messages/pl.json +++ b/messages/pl.json @@ -6711,7 +6711,7 @@ "baseUrl": "Adres", "allowedOrigins": "Dozwolone na tym serwerze: {origins}", "organizationId": "Identyfikator organizacji", - "organizationHelp": "Część po /organizations/ w pasku adresu Papra.", + "organizationHelp": "Część po /organizations/ w pasku adresu Papra. Identyfikatory organizacji Papra mają postać org_ i 24 litery oraz cyfry.", "token": "Token API", "tokenSaved": "Zapisano dla tego adresu. Zostaw puste, aby go zachować.", "tokenHelpPaperless": "W Paperless-ngx: Mój profil, token API.", @@ -10220,7 +10220,7 @@ "authRefused": "Archiwum odrzuciło token. Sprawdź token i jego uprawnienia.", "versionTooOld": "Archiwum jest za stare. Potrzebny jest Paperless-ngx 2.16 lub nowszy.", "badResponse": "Nie udało się odczytać odpowiedzi archiwum. Sprawdź, czy adres wskazuje na samo archiwum.", - "notFound": "Archiwum nie ma takiego dokumentu ani takiej organizacji.", + "notFound": "Archiwum nie ma takiego dokumentu albo identyfikator organizacji jest błędny lub nie należy do ciebie.", "linkTargetNotFound": "Wpis, z którym miało nastąpić powiązanie, już nie istnieje.", "rateLimited": "Za dużo żądań. Odczekaj minutę i spróbuj ponownie.", "browserOnly": "Z archiwów dokumentów można korzystać tylko w przeglądarce.", @@ -10233,7 +10233,8 @@ "uploadLimit": "Osiągnięto godzinny limit przesyłania. Spróbuj później.", "uploadBusy": "Trwają jeszcze inne przesyłania. Spróbuj za chwilę.", "moduleDisabled": "Dokumenty są wyłączone w ustawieniach.", - "generic": "Coś poszło nie tak. Spróbuj ponownie." + "generic": "Coś poszło nie tak. Spróbuj ponownie.", + "permissionMissing": "Klucz Papra nie może odczytać tagów. Nadaj mu uprawnienie tags:read." }, "pickMore": "Wybierz więcej", "finishedHint": "Wyszukaj ponownie lub zmień filtr, aby wybrać więcej dokumentów.", diff --git a/src/components/documents/sources/source-errors.ts b/src/components/documents/sources/source-errors.ts index 3c1739924..9ec63e293 100644 --- a/src/components/documents/sources/source-errors.ts +++ b/src/components/documents/sources/source-errors.ts @@ -22,6 +22,7 @@ export function isStoppingError(code: string | undefined): boolean { code === "documents.inbound.rateLimited" || code === "documents.inbound.quotaExceeded" || code === "documents.sources.authRefused" || + code === "documents.sources.permissionMissing" || code === "documents.sources.unreachable" || code === "documents.sources.originNotAllowed" || code === "documents.sources.notConnected" || @@ -45,6 +46,8 @@ export function sourceErrorMessage(t: T, code: string | undefined): string { return t("documents.sourcePicker.errors.redirected"); case "documents.sources.authRefused": return t("documents.sourcePicker.errors.authRefused"); + case "documents.sources.permissionMissing": + return t("documents.sourcePicker.errors.permissionMissing"); case "documents.sources.versionTooOld": return t("documents.sourcePicker.errors.versionTooOld"); case "documents.sources.badResponse": diff --git a/src/lib/documents/sources/__tests__/clients.test.ts b/src/lib/documents/sources/__tests__/clients.test.ts index 78f4f536b..ed6291335 100644 --- a/src/lib/documents/sources/__tests__/clients.test.ts +++ b/src/lib/documents/sources/__tests__/clients.test.ts @@ -41,7 +41,7 @@ const PAPRA = { system: "PAPRA" as const, origin: "https://papra.example.com", baseUrl: "https://papra.example.com", - organizationId: "org_123", + organizationId: "org_a1b2c3d4e5f6g7h8i9j0k1l2", token: "ppk_secret", }; @@ -383,6 +383,74 @@ describe("Paperless-ngx tags", () => { }); }); +describe("Papra, as it answers for real", () => { + it("quotes each word so a leading dash is not a negation, and sends the date range", async () => { + serve({ + "/api/organizations/org_a1b2c3d4e5f6g7h8i9j0k1l2/documents": () => + json({ documents: [], documentsCount: 0 }), + }); + await sourceClient(PAPRA).search({ + q: "-Knie Entlass", + tagId: null, + from: "2024-01-01", + to: "2024-12-31", + page: 1, + }); + const list = calledUrls().find((u) => u.pathname.endsWith("/documents")); + expect(list?.searchParams.get("searchQuery")).toBe( + '"-Knie" "Entlass" date:>=2024-01-01 date:<=2024-12-31', + ); + }); + + it("refuses an organization id Papra would not issue", () => { + for (const org of ["org_123", "../admin", "ORG_A1B2C3D4E5F6G7H8I9J0K1L2"]) { + expect(() => sourceClient({ ...PAPRA, organizationId: org })).toThrow( + DocumentSourceError, + ); + } + expect(safeFetch).not.toHaveBeenCalled(); + }); + + it("reads 'not in this organization' and a malformed id as no such organization", async () => { + serve({ + "/api/organizations/": () => + json({ error: { code: "user.not_in_organization" } }, { status: 403 }), + }); + expect( + ((await failure(sourceClient(PAPRA).test())) as DocumentSourceError).code, + ).toBe("notFound"); + serve({ + "/api/organizations/": () => + json( + { error: { code: "server.invalid_request.params" } }, + { status: 400 }, + ), + }); + expect( + ((await failure(sourceClient(PAPRA).test())) as DocumentSourceError).code, + ).toBe("notFound"); + // Any other refusal of the key is still the key. + serve({ + "/api/organizations/": () => + json({ error: { code: "auth.unauthorized" } }, { status: 401 }), + }); + expect( + ((await failure(sourceClient(PAPRA).test())) as DocumentSourceError).code, + ).toBe("authRefused"); + }); + + it("fails the connection test when the key cannot read tags", async () => { + serve({ + "/api/organizations/org_a1b2c3d4e5f6g7h8i9j0k1l2/tags": () => + json({}, { status: 401 }), + "/api/organizations/org_a1b2c3d4e5f6g7h8i9j0k1l2/documents": () => + json({ documents: [], documentsCount: 0 }), + }); + const err = await failure(sourceClient(PAPRA).test()); + expect((err as DocumentSourceError).code).toBe("permissionMissing"); + }); +}); + describe("Papra", () => { const tags = { tags: [ @@ -393,8 +461,8 @@ describe("Papra", () => { it("sends the name and the tag through searchQuery and re-checks both", async () => { serve({ - "/api/organizations/org_123/tags": () => json(tags), - "/api/organizations/org_123/documents": () => + "/api/organizations/org_a1b2c3d4e5f6g7h8i9j0k1l2/tags": () => json(tags), + "/api/organizations/org_a1b2c3d4e5f6g7h8i9j0k1l2/documents": () => json({ documentsCount: 60, documents: [ @@ -431,7 +499,7 @@ describe("Papra", () => { }); const list = calledUrls().find((u) => u.pathname.endsWith("/documents")); expect(list?.searchParams.get("searchQuery")).toBe( - 'blood tag:"Lab results"', + '"blood" tag:"Lab results" date:>=2025-01-01', ); expect(list?.searchParams.get("pageIndex")).toBe("0"); expect(list?.searchParams.get("pageSize")).toBe("25"); @@ -453,7 +521,9 @@ describe("Papra", () => { }); it("matches nothing for a tag that no longer exists", async () => { - serve({ "/api/organizations/org_123/tags": () => json(tags) }); + serve({ + "/api/organizations/org_a1b2c3d4e5f6g7h8i9j0k1l2/tags": () => json(tags), + }); await expect( sourceClient(PAPRA).search({ q: "", @@ -469,7 +539,9 @@ describe("Papra", () => { }); it("lists tags by name and maps an unknown organization to notFound", async () => { - serve({ "/api/organizations/org_123/tags": () => json(tags) }); + serve({ + "/api/organizations/org_a1b2c3d4e5f6g7h8i9j0k1l2/tags": () => json(tags), + }); await expect(sourceClient(PAPRA).tags()).resolves.toEqual([ { id: "tag_a", name: "Health" }, { id: "tag_b", name: "Lab results" }, @@ -482,9 +554,9 @@ describe("Papra", () => { it("reads a document and downloads its file", async () => { serve({ - "/api/organizations/org_123/documents/doc_1/file": () => - new Response(new Uint8Array([1, 2, 3])), - "/api/organizations/org_123/documents/doc_1": () => + "/api/organizations/org_a1b2c3d4e5f6g7h8i9j0k1l2/documents/doc_1/file": + () => new Response(new Uint8Array([1, 2, 3])), + "/api/organizations/org_a1b2c3d4e5f6g7h8i9j0k1l2/documents/doc_1": () => json({ document: { id: "doc_1", diff --git a/src/lib/documents/sources/clients.ts b/src/lib/documents/sources/clients.ts index a5d1cfa85..33d9371bf 100644 --- a/src/lib/documents/sources/clients.ts +++ b/src/lib/documents/sources/clients.ts @@ -30,8 +30,9 @@ import { DOCUMENT_PICKER_MAX_TAGS, DOCUMENT_PICKER_PAGE_SIZE, isSourceDocumentId, - PAPRA_ID, + PAPRA_ORG_ID, type DocumentPickerSystem, + type DocumentSourceErrorCode, type DocumentSourceTagDto, } from "./types"; @@ -335,13 +336,33 @@ function paperless(connection: SourceConnection): SourceClient { // ── Papra ────────────────────────────────────────────────────────────────── +/** + * Papra's error bodies carry a machine code. Two of them mean "no such + * organization" rather than "wrong key": a valid organization id the key's + * user is not a member of (403 `user.not_in_organization`), and one Papra + * cannot parse (400 `server.invalid_request.params`). + */ +function papraError( + status: number, + code: string | null, +): DocumentSourceErrorCode | null { + if (status === 403 && code === "user.not_in_organization") return "notFound"; + if (status === 400 && code === "server.invalid_request.params") { + return "notFound"; + } + return null; +} + /** Papra's search syntax quotes a tag name that holds a space. */ function papraTagTerm(name: string): string { return /[\s"]/.test(name) ? `tag:${JSON.stringify(name)}` : `tag:${name}`; } function papra(connection: SourceConnection): SourceClient { - if (!connection.organizationId || !PAPRA_ID.test(connection.organizationId)) { + if ( + !connection.organizationId || + !PAPRA_ORG_ID.test(connection.organizationId) + ) { throw new DocumentSourceError("notFound"); } const org = `/api/organizations/${encodeURIComponent(connection.organizationId)}`; @@ -352,6 +373,7 @@ function papra(connection: SourceConnection): SourceClient { Authorization: `Bearer ${connection.token}`, Accept: "application/json", }, + classify: papraError, }; const download: SourceTarget = { ...target, @@ -384,13 +406,31 @@ function papra(connection: SourceConnection): SourceClient { `${org}/documents?pageIndex=0&pageSize=1`, ); if (!asRecord(body)) throw shapeError(); + // The tag filter needs `tags:read`; a key without it would save green + // and then fail on the first search. + try { + await sourceJson(target, `${org}/tags`); + } catch (err) { + if (err instanceof DocumentSourceError && err.code === "authRefused") { + throw new DocumentSourceError( + "permissionMissing", + err.upstreamStatus, + ); + } + throw err; + } }, tags: listTags, async search(params) { - const terms: string[] = []; - if (params.q) terms.push(params.q); + // Each word quoted: Papra reads a leading `-` as a negation and a `:` as + // a field, and a quoted word still matches as a prefix. + const terms: string[] = params.q + .split(/\s+/) + .map((word) => word.replace(/"/g, "")) + .filter((word) => word.length > 0) + .map((word) => `"${word}"`); if (params.tagId) { const tag = (await listTags()).find((t) => t.id === params.tagId); // A tag that no longer exists matches nothing, rather than silently @@ -398,6 +438,10 @@ function papra(connection: SourceConnection): SourceClient { if (!tag) return { items: [], hasMore: false }; terms.push(papraTagTerm(tag.name)); } + // The range goes to Papra as its own `date:` filter; the check on each + // row below stays as the net for a version that reads it differently. + if (params.from) terms.push(`date:>=${params.from}`); + if (params.to) terms.push(`date:<=${params.to}`); const pageIndex = params.page - 1; const query = new URLSearchParams({ pageIndex: String(pageIndex), diff --git a/src/lib/documents/sources/http.ts b/src/lib/documents/sources/http.ts index 473d442ea..d5daf1b25 100644 --- a/src/lib/documents/sources/http.ts +++ b/src/lib/documents/sources/http.ts @@ -62,6 +62,8 @@ export interface SourceTarget { /** Origin plus the optional base path, no trailing slash. */ baseUrl: string; headers: Record; + /** How this source's error bodies read, when they say more than a status. */ + classify?: ErrorClassifier; } async function sourceRequest( @@ -103,13 +105,54 @@ async function discard(response: Response): Promise { } } +/** + * A source-specific reading of an error answer: given the status and the + * machine code in the body (if any), the error to report, or null for the + * generic mapping. The body itself is never passed on. + */ +export type ErrorClassifier = ( + status: number, + code: string | null, +) => DocumentSourceErrorCode | null; + +/** Longest error body read to find its machine code. */ +const ERROR_BODY_MAX_BYTES = 4 * 1024; + +/** The `code` (or `error.code`) of a JSON error body, if there is one. */ +async function errorCodeOf(response: Response): Promise { + try { + const bytes = await readBoundedBody(response.body, ERROR_BODY_MAX_BYTES); + const body = JSON.parse(new TextDecoder().decode(bytes)) as unknown; + if (!body || typeof body !== "object") return null; + const record = body as Record; + const nested = record.error as Record | undefined; + const code = + typeof record.code === "string" + ? record.code + : nested && + typeof nested === "object" && + typeof nested.code === "string" + ? nested.code + : null; + return code && code.length <= 100 ? code : null; + } catch { + return null; + } +} + /** Map a non-2xx answer to the error the person sees. */ export async function failFor( response: Response, notFound: DocumentSourceErrorCode = "notFound", + classify?: ErrorClassifier, ): Promise { - await discard(response); const status = response.status; + if (classify) { + const specific = classify(status, await errorCodeOf(response)); + if (specific) throw new DocumentSourceError(specific, status); + } else { + await discard(response); + } if (status === 401 || status === 403) { throw new DocumentSourceError("authRefused", status); } @@ -124,10 +167,19 @@ export async function failFor( export async function sourceJson( target: SourceTarget, path: string, - options: { notFound?: DocumentSourceErrorCode } = {}, + options: { + notFound?: DocumentSourceErrorCode; + classify?: ErrorClassifier; + } = {}, ): Promise<{ body: unknown; response: Response }> { const response = await sourceRequest(target, path, SOURCE_JSON_TIMEOUT_MS); - if (!response.ok) return failFor(response, options.notFound); + if (!response.ok) { + return failFor( + response, + options.notFound, + options.classify ?? target.classify, + ); + } const declared = Number(response.headers.get("content-length") ?? 0); if (declared > SOURCE_JSON_MAX_BYTES) { await discard(response); @@ -162,7 +214,7 @@ export async function sourceDownload( path, SOURCE_DOWNLOAD_TIMEOUT_MS, ); - if (!response.ok) return failFor(response); + if (!response.ok) return failFor(response, "notFound", target.classify); const declared = Number(response.headers.get("content-length") ?? 0); if (declared > maxFileBytes) { await discard(response); diff --git a/src/lib/documents/sources/route-support.ts b/src/lib/documents/sources/route-support.ts index a1b9aba86..126144674 100644 --- a/src/lib/documents/sources/route-support.ts +++ b/src/lib/documents/sources/route-support.ts @@ -83,6 +83,7 @@ const STATUS: Record = { unreachable: 502, redirected: 502, authRefused: 502, + permissionMissing: 502, versionTooOld: 502, badResponse: 502, notFound: 404, @@ -99,6 +100,8 @@ const MESSAGE: Record = { redirected: "The document archive redirected the request. Use the address it redirects to.", authRefused: "The document archive refused the API token.", + permissionMissing: + "The API key cannot read the archive's tags. Give it the tags:read permission.", versionTooOld: "The document archive's version is too old.", badResponse: "The document archive gave an answer HealthLog cannot read.", notFound: "The document archive has no such document or organization.", diff --git a/src/lib/documents/sources/types.ts b/src/lib/documents/sources/types.ts index 6dd8ddf35..3fdbfd1aa 100644 --- a/src/lib/documents/sources/types.ts +++ b/src/lib/documents/sources/types.ts @@ -43,8 +43,10 @@ export const DOCUMENT_PICKER_MAX_TAGS = 1000; /** Paperless-ngx document ids are integers. */ export const PAPERLESS_ID = /^\d{1,18}$/; -/** Papra document and organization ids: letters, digits, `_` and `-`. */ +/** Papra document ids: letters, digits, `_` and `-`. */ export const PAPRA_ID = /^[A-Za-z0-9_-]{1,128}$/; +/** Papra organization ids, as Papra issues them. */ +export const PAPRA_ORG_ID = /^org_[a-z0-9]{24}$/; /** * Is `id` a document id `system` could have issued? Checked before an id is @@ -134,6 +136,7 @@ export const DOCUMENT_SOURCE_ERROR_CODES = [ "documents.sources.unreachable", "documents.sources.redirected", "documents.sources.authRefused", + "documents.sources.permissionMissing", "documents.sources.versionTooOld", "documents.sources.badResponse", "documents.sources.notFound", diff --git a/src/lib/openapi/error-codes.ts b/src/lib/openapi/error-codes.ts index f8bbf0e7b..bb300d4ae 100644 --- a/src/lib/openapi/error-codes.ts +++ b/src/lib/openapi/error-codes.ts @@ -145,6 +145,7 @@ export const ERROR_CODE_CATALOGUE: Readonly> = "documents.sources.notFound", "documents.sources.organizationRequired", "documents.sources.originNotAllowed", + "documents.sources.permissionMissing", "documents.sources.rateLimited", "documents.sources.redirected", "documents.sources.tokenRequired", diff --git a/src/lib/openapi/routes/document-sources.ts b/src/lib/openapi/routes/document-sources.ts index 53f4cfe6f..0b97d283c 100644 --- a/src/lib/openapi/routes/document-sources.ts +++ b/src/lib/openapi/routes/document-sources.ts @@ -95,13 +95,13 @@ const refusal = { const unavailable = { description: - "`documents.sources.unavailable`: the operator has not listed any origin in `DOCUMENT_SOURCE_ORIGINS`, so the picker is off. `documents.sources.notConnected`: no connection is saved for this system. `documents.sources.notFound`: the source has no such document, or (Papra) no such organization.", + "`documents.sources.unavailable`: the operator has not listed any origin in `DOCUMENT_SOURCE_ORIGINS`, so the picker is off. `documents.sources.notConnected`: no connection is saved for this system. `documents.sources.notFound`: the source has no such document, or (Papra) no such organization, including one the key's user is not a member of (Papra 403 `user.not_in_organization`) and one Papra cannot parse (400).", content: { "application/json": { schema: errorEnvelope } }, }; const upstream = { description: - "The source could not answer usefully, with the reason in `meta.errorCode` and, when the source answered at all, its HTTP status in `meta.upstreamStatus` (its body is never passed on): `documents.sources.unreachable` (no connection, timeout, or a destination refused at dial time), `documents.sources.redirected` (the source answered with a redirect, which is never followed), `documents.sources.authRefused` (401/403 from the source), `documents.sources.versionTooOld` (Paperless-ngx older than 2.16 / API version 9), `documents.sources.badResponse` (not the documented JSON, or larger than 1 MiB).", + "The source could not answer usefully, with the reason in `meta.errorCode` and, when the source answered at all, its HTTP status in `meta.upstreamStatus` (its body is never passed on): `documents.sources.unreachable` (no connection, timeout, or a destination refused at dial time), `documents.sources.redirected` (the source answered with a redirect, which is never followed), `documents.sources.authRefused` (401/403 from the source), `documents.sources.permissionMissing` (Papra: the key cannot read tags; needs `tags:read`), `documents.sources.versionTooOld` (Paperless-ngx older than 2.16 / API version 9), `documents.sources.badResponse` (not the documented JSON, or larger than 1 MiB).", content: { "application/json": { schema: errorEnvelope } }, }; diff --git a/src/lib/validations/document-sources.ts b/src/lib/validations/document-sources.ts index d175c601c..4034d4c39 100644 --- a/src/lib/validations/document-sources.ts +++ b/src/lib/validations/document-sources.ts @@ -10,7 +10,7 @@ import { DOCUMENT_PICKER_LINK_KINDS, DOCUMENT_PICKER_MAX_PAGE, DOCUMENT_PICKER_QUERY_MAX, - PAPRA_ID, + PAPRA_ORG_ID, } from "@/lib/documents/sources/types"; import { documentSourceIdSchema, @@ -38,7 +38,7 @@ export const documentSourceSaveSchema = z.object({ organizationId: z .string() .trim() - .regex(PAPRA_ID, "Expected letters, digits, _ or -") + .regex(PAPRA_ORG_ID, "Expected a Papra organization id (org_…)") .optional(), token: z .string() From d69e12bac282ee40525f5aa0c714a0e5f5d85b3a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Sat, 26 Sep 2026 14:47:12 +0200 Subject: [PATCH 16/18] fix(scripts): skip documents the source already says are too large once HealthLog has named its limit, and show small sizes in KB --- scripts/__tests__/import-documents.test.ts | 37 ++++++++++++++++++++-- scripts/import-documents.mjs | 28 ++++++++++++++-- 2 files changed, 60 insertions(+), 5 deletions(-) diff --git a/scripts/__tests__/import-documents.test.ts b/scripts/__tests__/import-documents.test.ts index 994ded97e..754a41e38 100644 --- a/scripts/__tests__/import-documents.test.ts +++ b/scripts/__tests__/import-documents.test.ts @@ -343,7 +343,7 @@ async function papra() { return { status: 200, json: { - documentsCount: 3, + documentsCount: 4, documents: [ { id: "doc_a", @@ -365,6 +365,15 @@ async function papra() { { id: "tag_l", name: "Lab" }, ], }, + { + id: "doc_huge", + name: "Scan.pdf", + originalName: "scan.pdf", + originalSize: 40_000_000, + createdAt: "2025-03-10T12:00:00.000Z", + documentDate: "2020-03-10", + tags: [{ id: "tag_h", name: "Health" }], + }, { id: "doc_old", name: "Old.pdf", @@ -677,7 +686,31 @@ describe("import-documents.mjs — Papra", () => { ); expect(pp.seen[0].auth).toBe("Bearer pp_test"); expect(res.stdout).toMatch(/Imported: 1/); - expect(res.stdout).toMatch(/Skipped: 1/); + // doc_b answered 413 with the cap; doc_huge is known to be larger than + // that and is skipped without being downloaded or sent. + expect(res.stdout).toMatch(/Skipped: 2/); expect(res.stdout).toMatch(/larger than HealthLog accepts/); + expect(pp.seen.some((s) => s.path.includes("doc_huge"))).toBe(false); + }); + + it("shows small sizes in KB on a dry run", async () => { + const pp = await papra(); + const res = await runScript( + [ + "papra", + "--papra-url", + pp.url, + "--papra-org", + "org_1", + "--tag", + "Health", + "--since", + "2016-01-01", + "--dry-run", + ], + { PAPRA_TOKEN: "pp_test" }, + ); + expect(res.code).toBe(0); + expect(res.stdout).toMatch(/2\.0 KB\s+Papra doc_a/); }); }); diff --git a/scripts/import-documents.mjs b/scripts/import-documents.mjs index c909a35f2..194af9d24 100644 --- a/scripts/import-documents.mjs +++ b/scripts/import-documents.mjs @@ -647,6 +647,8 @@ async function upload(options, source, doc, file) { return { outcome: "skipped", reason: `larger than HealthLog accepts${meta.maxFileBytes ? ` (${mib(meta.maxFileBytes)})` : ""}`, + maxFileBytes: + typeof meta.maxFileBytes === "number" ? meta.maxFileBytes : undefined, }; case 415: return { @@ -668,9 +670,10 @@ async function upload(options, source, doc, file) { function mib(bytes) { if (typeof bytes !== "number") return "?"; - return bytes >= 1024 * 1024 * 1024 - ? `${(bytes / 1024 ** 3).toFixed(1)} GB` - : `${(bytes / 1024 ** 2).toFixed(1)} MB`; + if (bytes >= 1024 ** 3) return `${(bytes / 1024 ** 3).toFixed(1)} GB`; + // Under 0.1 MB a size in MB reads as 0.0; say it in KB. + if (bytes < 0.1 * 1024 ** 2) return `${(bytes / 1024).toFixed(1)} KB`; + return `${(bytes / 1024 ** 2).toFixed(1)} MB`; } // ── Run ──────────────────────────────────────────────────────────────────── @@ -705,6 +708,8 @@ async function run(argv, env = process.env) { }; let listed = 0; let totalBytes = 0; + /** HealthLog's per-file limit, once a 413 has said it. */ + let maxFileBytes = null; let unknownSizes = 0; out( @@ -743,6 +748,20 @@ async function run(argv, env = process.env) { continue; } + // HealthLog says its per-file limit only when it refuses a file. Once + // it has, a document the source already says is larger is skipped + // without downloading it. + if ( + maxFileBytes !== null && + typeof doc.size === "number" && + doc.size > maxFileBytes + ) { + const reason = `larger than HealthLog accepts (${mib(maxFileBytes)})`; + tally.skipped.push({ where, reason }); + out(` skipped ${where}: ${reason}`); + continue; + } + const file = await source.download(doc); if (file.skip) { tally.skipped.push({ where, reason: file.skip }); @@ -750,6 +769,9 @@ async function run(argv, env = process.env) { continue; } const result = await upload(options, source, doc, file); + if (typeof result.maxFileBytes === "number") { + maxFileBytes = result.maxFileBytes; + } if (result.outcome === "imported") { tally.imported++; out(` imported ${where}`); From bb5d0ab933d3593b23149b951e92a8d558666e66 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Sat, 26 Sep 2026 14:47:40 +0200 Subject: [PATCH 17/18] docs(documents): Papra organization ids and tags permission, deletion by content, the Paperless SQLite case limit --- docs/integrations/document-import.md | 11 +++++++++-- docs/self-hosting/document-sources.md | 13 ++++++++++++- 2 files changed, 21 insertions(+), 3 deletions(-) diff --git a/docs/integrations/document-import.md b/docs/integrations/document-import.md index 0c5c54628..d7a0163f1 100644 --- a/docs/integrations/document-import.md +++ b/docs/integrations/document-import.md @@ -152,8 +152,15 @@ What it does: quick. Documents you deleted in HealthLog stay deleted, also after they are removed for good. The same goes for a file you had already uploaded by hand: the import recognises it by its content (sending it once to - compare), and if you delete it later, the import leaves it deleted. An - interrupted run simply picks up where it stopped. + compare), and if you delete it later, the import leaves it deleted. Since + v1.39.3 a deleted document is also recognised by its content while it is + in the 30-day undo window: the same file arriving from another system or + under a new id is answered "deleted" and not stored again. After the purge + HealthLog keeps the ids it saw, not the content. An interrupted run simply + picks up where it stopped. +- Once HealthLog has refused one file as too large, the script skips any + further document the source already lists as larger (Papra lists sizes) + without downloading it. - When HealthLog asks it to slow down, it waits as long as HealthLog says and carries on. By default a document token may send 120 uploads an hour, so an archive of a thousand documents takes a night. Every upload that diff --git a/docs/self-hosting/document-sources.md b/docs/self-hosting/document-sources.md index 8b35367a0..4a30cdd2c 100644 --- a/docs/self-hosting/document-sources.md +++ b/docs/self-hosting/document-sources.md @@ -49,7 +49,9 @@ Settings → Integrations → Document archives: - **Paperless-ngx:** the base address and an API token (Paperless-ngx: My Profile → API token). Paperless-ngx 2.16 or later (API version 9). - **Papra:** the base address, the organization id (from the Papra address - bar) and an API key with `documents:read` and `tags:read`. + bar, `org_` followed by 24 letters and digits) and an API key with + `documents:read` and `tags:read`. The connection test reads the tags too, + so a key without `tags:read` is refused at save. HealthLog tests the connection before saving it. A saved token is only ever sent to the address it was saved for: changing the address to another origin @@ -69,6 +71,15 @@ limit for a document). ## Limits and rules +- Name search on Paperless-ngx uses its own title filter. On a Paperless-ngx + that stores its data in SQLite, that filter ignores upper and lower case + only for plain ASCII letters: searching `röntgen` does not find `Röntgen`. + That is Paperless-ngx's behaviour, not HealthLog's; PostgreSQL-backed + Paperless does not have it. +- A document deleted in HealthLog is recognised by its content during the + 30-day undo window, so the same file from another archive is not stored + again; after the purge only its ids are remembered. + - Imports count against the person's normal upload allowance (60 an hour) and storage quota. - A document that is already in HealthLog is not stored twice, and one the From 656542b0401275d7d13cfc7f7408a5305c79c36e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Sat, 26 Sep 2026 15:15:32 +0200 Subject: [PATCH 18/18] fix(documents): date Papra search rows by their document date only Papra's date: filter matches the document date, so a row now shows that date or none, and an undated document falls out of a date range in the list as it does on the server. Import still files an undated document under the day it was added. --- .../sources/__tests__/clients.test.ts | 63 +++++++++++++++++++ src/lib/documents/sources/clients.ts | 9 ++- 2 files changed, 69 insertions(+), 3 deletions(-) diff --git a/src/lib/documents/sources/__tests__/clients.test.ts b/src/lib/documents/sources/__tests__/clients.test.ts index ed6291335..bf6850942 100644 --- a/src/lib/documents/sources/__tests__/clients.test.ts +++ b/src/lib/documents/sources/__tests__/clients.test.ts @@ -402,6 +402,68 @@ describe("Papra, as it answers for real", () => { ); }); + it("dates a search row by its document date only, as Papra's date filter does", async () => { + serve({ + "/api/organizations/org_a1b2c3d4e5f6g7h8i9j0k1l2/documents": () => + json({ + documentsCount: 2, + documents: [ + { + id: "doc_dated", + name: "A.pdf", + documentDate: "2024-05-01", + createdAt: "2025-01-01T00:00:00.000Z", + }, + { + id: "doc_undated", + name: "B.pdf", + createdAt: "2025-01-01T00:00:00.000Z", + }, + ], + }), + }); + const all = await sourceClient(PAPRA).search({ + q: "", + tagId: null, + from: null, + to: null, + page: 1, + }); + expect(all.items.map((i) => [i.sourceId, i.date])).toEqual([ + ["doc_dated", "2024-05-01"], + ["doc_undated", null], + ]); + const ranged = await sourceClient(PAPRA).search({ + q: "", + tagId: null, + from: "2024-01-01", + to: "2025-12-31", + page: 1, + }); + // Papra's date: filter never matches a document without a document date; + // the per-row check agrees instead of re-admitting it by upload day. + expect(ranged.items.map((i) => i.sourceId)).toEqual(["doc_dated"]); + }); + + it("still files an undated Papra document under the day it was added on import", async () => { + serve({ + "/api/organizations/org_a1b2c3d4e5f6g7h8i9j0k1l2/documents/doc_undated": + () => + json({ + document: { + id: "doc_undated", + name: "B.pdf", + createdAt: "2025-01-01T09:00:00.000Z", + }, + }), + }); + await expect( + sourceClient(PAPRA).document("doc_undated"), + ).resolves.toMatchObject({ + date: "2025-01-01", + }); + }); + it("refuses an organization id Papra would not issue", () => { for (const org of ["org_123", "../admin", "ORG_A1B2C3D4E5F6G7H8I9J0K1L2"]) { expect(() => sourceClient({ ...PAPRA, organizationId: org })).toThrow( @@ -471,6 +533,7 @@ describe("Papra", () => { name: "Blood test.pdf", originalName: "Blood test.pdf", originalSize: 1234, + documentDate: "2025-03-04", createdAt: "2025-03-04T09:00:00.000Z", tags: [{ id: "tag_b", name: "Lab results" }], }, diff --git a/src/lib/documents/sources/clients.ts b/src/lib/documents/sources/clients.ts index 33d9371bf..5580e9f65 100644 --- a/src/lib/documents/sources/clients.ts +++ b/src/lib/documents/sources/clients.ts @@ -464,9 +464,12 @@ function papra(connection: SourceConnection): SourceClient { if (params.tagId && !tags.some((t) => asText(t.id) === params.tagId)) { continue; } - // Papra's search date syntax differs between versions; the range is - // applied here instead of trusting a query syntax. - const date = dayOf(doc.documentDate) ?? dayOf(doc.createdAt); + // The row shows the document date only, the field Papra's `date:` + // filter reads, so the list and the filter agree: an undated document + // shows no date and falls out of any range. The check below is the net + // for a server that ignores the filter. The upload day is used only + // on import, where a filing date is needed (see `document()`). + const date = dayOf(doc.documentDate); if (!withinRange(date, params.from, params.to)) continue; const size = doc.originalSize; items.push({