diff --git a/prisma/migrations/0356_document_source_connections/migration.sql b/prisma/migrations/0356_document_source_connections/migration.sql index 6423236ed..f50be788c 100644 --- a/prisma/migrations/0356_document_source_connections/migration.sql +++ b/prisma/migrations/0356_document_source_connections/migration.sql @@ -20,3 +20,27 @@ CREATE UNIQUE INDEX "document_source_connections_user_id_system_key" ALTER TABLE "document_source_connections" ADD CONSTRAINT "document_source_connections_user_id_fkey" FOREIGN KEY ("user_id") REFERENCES "users"("id") ON DELETE CASCADE ON UPDATE CASCADE; + +-- Source instance (#1038 review): the same id in two Paperless-ngx (or two +-- Papra) instances is two documents. A nullable origin joins every source key; +-- keys stored without one (v1.39.2 imports) keep matching any instance, which +-- the application decides at lookup time. NULLS NOT DISTINCT keeps two keys +-- without an instance colliding exactly as before (PG15+; the stack runs 16). +ALTER TABLE "inbound_documents" ADD COLUMN "source_instance" VARCHAR(512); +DROP INDEX IF EXISTS "inbound_documents_user_source_key"; +CREATE UNIQUE INDEX "inbound_documents_user_source_key" + ON "inbound_documents" ("user_id", "source_system", "source_instance", "source_id") + NULLS NOT DISTINCT + WHERE "source_id" IS NOT NULL; + +ALTER TABLE "document_source_aliases" ADD COLUMN "source_instance" VARCHAR(512); +DROP INDEX IF EXISTS "document_source_aliases_user_id_source_system_source_id_key"; +CREATE UNIQUE INDEX "document_source_aliases_user_id_source_system_source_instance_source_id_key" + ON "document_source_aliases" ("user_id", "source_system", "source_instance", "source_id") + NULLS NOT DISTINCT; + +ALTER TABLE "document_import_keys" ADD COLUMN "source_instance" VARCHAR(512); +DROP INDEX IF EXISTS "document_import_keys_user_id_source_system_source_id_key"; +CREATE UNIQUE INDEX "document_import_keys_user_id_source_system_source_instance_source_id_key" + ON "document_import_keys" ("user_id", "source_system", "source_instance", "source_id") + NULLS NOT DISTINCT; diff --git a/prisma/schema.prisma b/prisma/schema.prisma index 47037b890..6c6c06bca 100644 --- a/prisma/schema.prisma +++ b/prisma/schema.prisma @@ -8011,15 +8011,24 @@ model InboundDocument { /// and the Paperless workflow recipe; shown on the detail sheet. sourceSystem String? @map("source_system") @db.VarChar(16) /// v1.39.2 (#1038) — the document's id in `sourceSystem`. Together with - /// `userId` + `sourceSystem` it is unique across live AND tombstoned rows - /// (raw SQL in migration 0353, not expressible in Prisma): - /// CREATE UNIQUE INDEX inbound_documents_user_source_key - /// ON inbound_documents (user_id, source_system, source_id) - /// WHERE source_id IS NOT NULL; + /// `userId`, `sourceSystem` and `sourceInstance` it is unique across live + /// AND tombstoned rows (raw SQL in migrations 0353 and 0356, not expressible + /// in Prisma; see `sourceInstance` for the current definition). /// Including tombstones is the point: re-running an importer must not bring /// back a document the person deleted. The purge carries the key over to /// `DocumentImportKey` when it removes the row. sourceId String? @map("source_id") @db.VarChar(128) + /// v1.39.3 (#1038) — which instance of `sourceSystem` the document came + /// from: the normalised origin (`scheme://host[:port]`) of the Paperless-ngx + /// or Papra it was fetched from, so id 123 on one Paperless and id 123 on + /// another are two keys. Plaintext; an address, not health data. Null for a + /// key sent without one (every v1.39.2 import, an older script, a workflow + /// that does not send it): such a key matches any instance of its system, + /// and a lookup without an instance matches any stored instance. Part of the + /// source-key unique index since migration 0356, which rebuilt it as + /// (user_id, source_system, source_instance, source_id) NULLS NOT DISTINCT + /// WHERE source_id IS NOT NULL. + sourceInstance String? @map("source_instance") @db.VarChar(512) /// v1.39.2 (#1038) — the upload asked for `aiRead=defer`: nothing may send /// this document to an AI provider on its own (the summary catch-up, the /// summary job, lab auto-staging all skip it). Cleared when the person @@ -8081,18 +8090,24 @@ model InboundDocument { /// own, so deleting the document keeps every key that ever reached it deleted. /// Cascades with the document; the purge copies it first. model DocumentSourceAlias { - id String @id @default(cuid()) - userId String @map("user_id") - documentId String @map("document_id") + id String @id @default(cuid()) + userId String @map("user_id") + documentId String @map("document_id") /// Same closed vocabulary as `InboundDocument.sourceSystem`. - sourceSystem String @map("source_system") @db.VarChar(16) - sourceId String @map("source_id") @db.VarChar(128) - createdAt DateTime @default(now()) @map("created_at") + sourceSystem String @map("source_system") @db.VarChar(16) + sourceId String @map("source_id") @db.VarChar(128) + /// v1.39.3 — the source instance, as on `InboundDocument.sourceInstance` + /// (null matches any instance of the system). + sourceInstance String? @map("source_instance") @db.VarChar(512) + createdAt DateTime @default(now()) @map("created_at") user User @relation(fields: [userId], references: [id], onDelete: Cascade) document InboundDocument @relation(fields: [documentId], references: [id], onDelete: Cascade) - @@unique([userId, sourceSystem, sourceId]) + // NULLS NOT DISTINCT in migration 0356 (hand-written; Prisma would + // regenerate it NULLS DISTINCT): a key stored without an instance still + // collides with the same key without one. + @@unique([userId, sourceSystem, sourceInstance, sourceId]) @@index([documentId]) @@map("document_source_aliases") } @@ -8104,16 +8119,22 @@ model DocumentSourceAlias { /// "deleted" instead of storing the document again. Written by the purge only; /// read by the upload route's source-key check. Holds no document content. model DocumentImportKey { - id String @id @default(cuid()) - userId String @map("user_id") + id String @id @default(cuid()) + userId String @map("user_id") /// Same closed vocabulary as `InboundDocument.sourceSystem`. - sourceSystem String @map("source_system") @db.VarChar(16) - sourceId String @map("source_id") @db.VarChar(128) - createdAt DateTime @default(now()) @map("created_at") + sourceSystem String @map("source_system") @db.VarChar(16) + sourceId String @map("source_id") @db.VarChar(128) + /// v1.39.3 — the source instance, as on `InboundDocument.sourceInstance` + /// (null matches any instance of the system). + sourceInstance String? @map("source_instance") @db.VarChar(512) + createdAt DateTime @default(now()) @map("created_at") user User @relation(fields: [userId], references: [id], onDelete: Cascade) - @@unique([userId, sourceSystem, sourceId]) + // NULLS NOT DISTINCT in migration 0356 (hand-written; Prisma would + // regenerate it NULLS DISTINCT): a key stored without an instance still + // collides with the same key without one. + @@unique([userId, sourceSystem, sourceInstance, sourceId]) @@map("document_import_keys") } diff --git a/scripts/__tests__/import-documents.test.ts b/scripts/__tests__/import-documents.test.ts index 6c5865fa4..994ded97e 100644 --- a/scripts/__tests__/import-documents.test.ts +++ b/scripts/__tests__/import-documents.test.ts @@ -424,6 +424,7 @@ describe("import-documents.mjs — Paperless-ngx", () => { kind: "LAB_RESULT", sourceSystem: "PAPERLESS", sourceId: "1", + sourceInstance: new URL(pl.url).origin, aiRead: "defer", }, }); @@ -664,6 +665,7 @@ describe("import-documents.mjs — Papra", () => { documentDate: "2019-06-01", sourceSystem: "PAPRA", sourceId: "doc_a", + sourceInstance: new URL(pp.url).origin, }); // --ai-read: no deferral field at all. expect(sent[0].aiRead).toBeUndefined(); diff --git a/scripts/import-documents.mjs b/scripts/import-documents.mjs index 1394136b3..c909a35f2 100644 --- a/scripts/import-documents.mjs +++ b/scripts/import-documents.mjs @@ -301,6 +301,7 @@ function paperless(options) { return { system: "PAPERLESS", + instance: new URL(base).origin, label, async *documents() { const query = new URLSearchParams({ @@ -324,7 +325,8 @@ function paperless(options) { tagId = found[0].id; query.set("tags__id__all", String(tagId)); } - if (options.since) query.set("created__date__gte", options.since); + // API version 9 made `created` a plain date: filter on it directly. + if (options.since) query.set("created__gte", options.since); let typeNames = new Map(); if (options.kindMap.size > 0) { @@ -399,6 +401,7 @@ function papra(options) { return { system: "PAPRA", + instance: new URL(options.baseUrl).origin, label, async *documents() { let tagId = null; @@ -526,6 +529,7 @@ function keyQuery(source, doc) { return new URLSearchParams({ sourceSystem: source.system, sourceId: doc.sourceId.slice(0, SOURCE_ID_MAX), + sourceInstance: source.instance, }); } @@ -617,6 +621,9 @@ async function upload(options, source, doc, file) { if (!options.aiRead) form.append("aiRead", "defer"); form.append("sourceSystem", source.system); form.append("sourceId", doc.sourceId.slice(0, SOURCE_ID_MAX)); + // Which Paperless-ngx or Papra this id belongs to: the same id in two + // instances is two documents. HealthLog 1.39.2 ignores the field. + form.append("sourceInstance", source.instance); return form; }, ); diff --git a/src/app/api/documents/inbound/__tests__/route.test.ts b/src/app/api/documents/inbound/__tests__/route.test.ts index bcb479701..a83d87578 100644 --- a/src/app/api/documents/inbound/__tests__/route.test.ts +++ b/src/app/api/documents/inbound/__tests__/route.test.ts @@ -51,10 +51,10 @@ vi.mock("@/lib/db", () => { findFirst: vi.fn(), }, documentImportKey: { - findUnique: vi.fn(), + findFirst: vi.fn(), }, documentSourceAlias: { - findUnique: vi.fn(), + findFirst: vi.fn(), createMany: vi.fn(), count: vi.fn(), }, @@ -239,10 +239,10 @@ beforeEach(() => { [] as never, ); vi.mocked(prisma.inboundDocument.findFirst).mockResolvedValue(null as never); - vi.mocked(prisma.documentImportKey.findUnique).mockResolvedValue( + vi.mocked(prisma.documentImportKey.findFirst).mockResolvedValue( null as never, ); - vi.mocked(prisma.documentSourceAlias.findUnique).mockResolvedValue( + vi.mocked(prisma.documentSourceAlias.findFirst).mockResolvedValue( null as never, ); vi.mocked(prisma.documentSourceAlias.createMany).mockResolvedValue({ @@ -783,7 +783,7 @@ describe("POST /api/documents/inbound — document import", () => { }); it("remembers a purged import through the ledger", async () => { - vi.mocked(prisma.documentImportKey.findUnique).mockResolvedValue({ + vi.mocked(prisma.documentImportKey.findFirst).mockResolvedValue({ id: "key-1", } as never); const res = await post( @@ -909,6 +909,8 @@ describe("POST /api/documents/inbound — document import", () => { userId: "user-1", documentId: "doc-manual", sourceSystem: "PAPRA", + // Sent without an instance: the alias matches any Papra. + sourceInstance: null, sourceId: "doc_9", }, ], @@ -936,7 +938,7 @@ describe("POST /api/documents/inbound — document import", () => { }); it("resolves a key held as an alias of a deleted document to deleted", async () => { - vi.mocked(prisma.documentSourceAlias.findUnique).mockResolvedValue({ + vi.mocked(prisma.documentSourceAlias.findFirst).mockResolvedValue({ document: docRow({ id: "doc-gone", deletedAt: new Date() }), } as never); const res = await post( diff --git a/src/app/api/documents/inbound/route.ts b/src/app/api/documents/inbound/route.ts index 90c4dde70..f2c6723d4 100644 --- a/src/app/api/documents/inbound/route.ts +++ b/src/app/api/documents/inbound/route.ts @@ -274,14 +274,17 @@ async function processUpload( const url = new URL(request.url); const querySystem = url.searchParams.get("sourceSystem"); const queryId = url.searchParams.get("sourceId"); + const queryInstance = url.searchParams.get("sourceInstance"); let queryKey: { sourceSystem: DocumentSourceSystemValue; sourceId: string; + sourceInstance?: string; } | null = null; if (querySystem !== null || queryId !== null) { const parsedKey = documentSourceKeySchema.safeParse({ sourceSystem: querySystem ?? undefined, sourceId: queryId ?? undefined, + sourceInstance: queryInstance ?? undefined, }); if (!parsedKey.success) { return apiValidationError( @@ -312,6 +315,7 @@ async function processUpload( user.id, queryKey.sourceSystem, queryKey.sourceId, + queryKey.sourceInstance ?? null, ); if (match) { return answerSourceKey( @@ -408,6 +412,7 @@ async function processUpload( encounterIds: rawEncounterIds.length > 0 ? rawEncounterIds : undefined, sourceSystem: formData.get("sourceSystem") ?? undefined, sourceId: formData.get("sourceId") ?? undefined, + sourceInstance: formData.get("sourceInstance") ?? undefined, aiRead: formData.get("aiRead") ?? undefined, }); if (!parsed.success) { @@ -425,10 +430,14 @@ async function processUpload( // both they have to agree. const formSystem = parsed.data.sourceSystem ?? null; const formId = formSystem ? (parsed.data.sourceId ?? null) : null; + const formInstance = formId ? (parsed.data.sourceInstance ?? null) : null; if ( queryKey && formSystem !== null && - (formSystem !== queryKey.sourceSystem || formId !== queryKey.sourceId) + (formSystem !== queryKey.sourceSystem || + formId !== queryKey.sourceId || + (formInstance !== null && + formInstance !== (queryKey.sourceInstance ?? null))) ) { return apiError( "The source key in the address and in the form differ.", @@ -438,6 +447,9 @@ async function processUpload( } const sourceSystem = queryKey?.sourceSystem ?? formSystem; const sourceId = queryKey?.sourceId ?? formId; + const sourceInstance = sourceId + ? (queryKey?.sourceInstance ?? formInstance ?? null) + : null; const ctx: UploadContext = { userId: user.id, scoped, @@ -451,7 +463,12 @@ async function processUpload( // count (the unique index has no `deleted_at` predicate), and past the purge // the ledger remembers. if (sourceSystem && sourceId && !queryKey) { - const match = await findSourceKey(user.id, sourceSystem, sourceId); + const match = await findSourceKey( + user.id, + sourceSystem, + sourceId, + sourceInstance, + ); if (match) return answerSourceKey(ctx, match); } @@ -478,6 +495,7 @@ async function processUpload( encounterIds: parsed.data.encounterIds ?? [], sourceSystem, sourceId, + sourceInstance, aiDeferred: ctx.aiDeferred, sourceKeyChecked: true, limits, diff --git a/src/app/api/documents/inbound/source/route.ts b/src/app/api/documents/inbound/source/route.ts index caada8b66..ec675a97c 100644 --- a/src/app/api/documents/inbound/source/route.ts +++ b/src/app/api/documents/inbound/source/route.ts @@ -57,6 +57,7 @@ export const GET = apiHandler(async (request: Request) => { const parsed = documentSourceKeySchema.safeParse({ sourceSystem: url.searchParams.get("sourceSystem") ?? undefined, sourceId: url.searchParams.get("sourceId") ?? undefined, + sourceInstance: url.searchParams.get("sourceInstance") ?? undefined, }); if (!parsed.success) { return apiValidationError( @@ -71,6 +72,7 @@ export const GET = apiHandler(async (request: Request) => { user.id, parsed.data.sourceSystem, parsed.data.sourceId, + parsed.data.sourceInstance ?? null, ); annotate({ action: { name: "documents.vault.sourceLookup" }, diff --git a/src/app/api/documents/sources/[system]/import/route.ts b/src/app/api/documents/sources/[system]/import/route.ts index 606e3c947..7da5933af 100644 --- a/src/app/api/documents/sources/[system]/import/route.ts +++ b/src/app/api/documents/sources/[system]/import/route.ts @@ -12,13 +12,16 @@ * imported (or already holding this key as an alias) is a `duplicate`, * and one the person deleted here is `deleted` — nothing is fetched and * nothing is stored, exactly as the token upload answers the same key; - * 3. only then the vault's own upload allowance (60 an hour, shared with - * uploads from the web and the phone) and an upload slot are taken, the - * document's details and its original file are fetched (bounded by the - * vault's per-file cap), and the file goes through `ingestDocument`, the - * same function the upload route stores through: type check, content - * dedup, quota, provenance (`sourceSystem` / `sourceId`), the background - * jobs; + * 3. only then is the source asked (on the search allowance, like every + * request to it) and an upload slot taken. One import per key and person + * runs at a time under an advisory lock, so a double click downloads the + * document once. The details and the original file are fetched (bounded + * by the vault's per-file cap); only a file that arrived is charged to + * the vault's own upload allowance (60 an hour, shared with uploads from + * the web and the phone), and it goes through `ingestDocument`, the same + * function the upload route stores through: type check, content dedup, + * quota, provenance (`sourceSystem` / `sourceId` / `sourceInstance`), the + * background jobs; * 4. the document is linked to the record through `src/lib/links/`. * * AI reading follows the person's own setting, unlike the token upload's @@ -48,18 +51,26 @@ import { UPLOAD_LIMIT_PER_HOUR, UPLOAD_WINDOW_MS, } from "@/lib/documents/ingest"; -import { findSourceKey, type SourceKeyMatch } from "@/lib/documents/source-key"; +import { + checkSourceLookupRateLimit, + findSourceKey, + type SourceKeyMatch, +} from "@/lib/documents/source-key"; +import type { SourceClient } from "@/lib/documents/sources/clients"; import { clientFor, loadConnection } from "@/lib/documents/sources/connections"; import { admitDocumentSourceCaller, + checkSearchRateLimit, responseForSourceFailure, sourceErrorResponse, systemParam, } from "@/lib/documents/sources/route-support"; -import type { - DocumentImportOutcome, - DocumentPickerLinkKind, - DocumentSourceImportDto, +import { + isSourceDocumentId, + type DocumentImportOutcome, + type DocumentPickerLinkKind, + type DocumentPickerSystem, + type DocumentSourceImportDto, } from "@/lib/documents/sources/types"; import { acquireDocumentUploadSlot, @@ -69,11 +80,17 @@ import { linkTargets } from "@/lib/links"; import { annotate } from "@/lib/logging/context"; import { checkRateLimit, rateLimitHeaders } from "@/lib/rate-limit"; import { documentSourceImportSchema } from "@/lib/validations/document-sources"; +import type { InboundDocumentKindValue } from "@/lib/validations/inbound-documents"; export const dynamic = "force-dynamic"; type RouteParams = { params: Promise<{ system: string }> }; +/** How long an import waits for a database connection to hold its lock on. */ +const IMPORT_LOCK_WAIT_MS = 10_000; +/** Longest a single import may hold its key: the download plus the store. */ +const IMPORT_LOCK_HOLD_MS = 150_000; + interface LinkTarget { kind: DocumentPickerLinkKind; id: string; @@ -172,6 +189,10 @@ export const POST = apiHandler( ); } const { sourceId, kind, link } = parsed.data; + // Only an id this system could have issued goes into a request path. + if (!isSourceDocumentId(system, sourceId)) { + return apiError("Not a document id of this archive.", 422); + } const row = await loadConnection(userId, system); if (!row) return sourceErrorResponse("notConnected"); @@ -181,25 +202,38 @@ export const POST = apiHandler( } catch (err) { return responseForSourceFailure(err); } + const instance = client.instance; if (link && !(await ownsLinkTarget(userId, link))) { return sourceErrorResponse("linkTargetNotFound"); } - // The source key answers before anything is downloaded or charged. - const held = await findSourceKey(userId, system, sourceId); + // The source key answers before anything is downloaded or charged, on the + // same metered lookup allowance the upload's key check draws on. + const lookupRl = await checkSourceLookupRateLimit( + false, + auth.session.id, + userId, + ); + if (!lookupRl.allowed) { + return sourceErrorResponse( + "rateLimited", + undefined, + rateLimitHeaders(lookupRl), + ); + } + const held = await findSourceKey(userId, system, sourceId, instance); if (held) return answerHeldKey(system, userId, link, held); - const rl = await checkRateLimit( - personalUploadBucket(userId), - UPLOAD_LIMIT_PER_HOUR, - UPLOAD_WINDOW_MS, - ); - if (!rl.allowed) { - return apiError("Too many uploads. Try again later.", 429, { - errorCode: "documents.inbound.rateLimited", - headers: rateLimitHeaders(rl), - }); + // A request to the source counts against the search allowance like any + // other; the upload allowance is charged only once a file arrived. + const fetchRl = await checkSearchRateLimit(userId); + if (!fetchRl.allowed) { + return sourceErrorResponse( + "rateLimited", + undefined, + rateLimitHeaders(fetchRl), + ); } const release = acquireDocumentUploadSlot(userId); @@ -216,83 +250,136 @@ export const POST = apiHandler( } try { - const limits = await resolveDocumentLimits(userId); - let meta; - let file; - try { - meta = await client.document(sourceId); - file = await client.download(sourceId, limits.maxFileBytes); - } catch (err) { - return responseForSourceFailure(err); - } - - const result = await ingestDocument({ - userId, - scoped: false, - ipAddress: getClientIp(request), - bytes: Buffer.from(file.bytes), - filename: meta.filename ?? file.filename, - title: meta.title, - kind: kind ?? null, - documentDate: meta.date, - episodeIds: [], - encounterIds: [], - sourceSystem: system, - sourceId, - aiDeferred: false, - sourceKeyChecked: true, - limits, - // The import is data; only the summary queued after a fresh insert - // follows the capability (and the person's auto-read setting). - documentAi: () => getAiCapability("documentAi"), - }); - - switch (result.kind) { - case "sourceKey": - return answerHeldKey(system, userId, link, result.match); - case "duplicate": { - // The same bytes were already in the vault (an upload, or another - // archive); the key is now remembered for that document. - const linked = await linkDocument(userId, link, result.document.id); - return respond(system, "duplicate", result.document.id, linked); - } - case "stored": { - const linked = await linkDocument(userId, link, result.document.id); - return respond(system, "imported", result.document.id, linked); - } - case "tooLarge": - return apiError("File is too large.", 413, { - errorCode: "documents.inbound.fileTooLarge", - reason: "fileTooLarge", - maxFileBytes: result.maxFileBytes, + // One import of one key at a time per person: a second tab or a double + // click waits here, then finds the key the first one stored, so the + // document is downloaded once. The lock is transaction-scoped and the + // transaction holds nothing else; the store runs in its own. + return await prisma.$transaction( + async (tx) => { + await tx.$queryRaw` + SELECT 1 AS locked + FROM pg_advisory_xact_lock(hashtextextended(${`documents-import:${userId}:${system}:${instance}:${sourceId}`}, 0)) + `; + const raced = await findSourceKey(userId, system, sourceId, instance); + if (raced) return answerHeldKey(system, userId, link, raced); + return importOne({ + request, + client, + userId, + system, + instance, + sourceId, + kind: kind ?? null, + link, }); - case "empty": - case "unsupportedType": - return apiError("This file type is not supported.", 415, { - errorCode: "documents.inbound.fileType", - reason: "unsupportedType", - }); - case "quotaExceeded": - return apiError("Storage quota exceeded.", 413, { - errorCode: "documents.inbound.quotaExceeded", - reason: "quotaExceeded", - quotaBytes: result.quotaBytes, - usedBytes: result.usedBytes, - }); - case "aliasLimit": - return apiError( - "This file is already stored under too many source ids.", - 409, - { errorCode: "documents.inbound.sourceAliasLimit" }, - ); - case "episodeNotFound": - case "encounterNotFound": - // No pre-links are passed on this path; unreachable, but a result - // the switch does not name must not fall through silently. - return sourceErrorResponse("linkTargetNotFound"); - } + }, + { maxWait: IMPORT_LOCK_WAIT_MS, timeout: IMPORT_LOCK_HOLD_MS }, + ); } finally { release(); } }, ); + +/** Fetch one document and store it; the caller holds the key's lock. */ +async function importOne(args: { + request: Request; + client: SourceClient; + userId: string; + system: DocumentPickerSystem; + instance: string; + sourceId: string; + kind: InboundDocumentKindValue | null; + link: LinkTarget | undefined; +}): Promise { + const { request, client, userId, system, instance, sourceId, kind, link } = + args; + const limits = await resolveDocumentLimits(userId); + let meta; + let file; + try { + meta = await client.document(sourceId); + file = await client.download(sourceId, limits.maxFileBytes); + } catch (err) { + return responseForSourceFailure(err); + } + + const rl = await checkRateLimit( + personalUploadBucket(userId), + UPLOAD_LIMIT_PER_HOUR, + UPLOAD_WINDOW_MS, + ); + if (!rl.allowed) { + return apiError("Too many uploads. Try again later.", 429, { + errorCode: "documents.inbound.rateLimited", + headers: rateLimitHeaders(rl), + }); + } + + const result = await ingestDocument({ + userId, + scoped: false, + ipAddress: getClientIp(request), + bytes: Buffer.from(file.bytes), + filename: meta.filename ?? file.filename, + title: meta.title, + kind, + documentDate: meta.date, + episodeIds: [], + encounterIds: [], + sourceSystem: system, + sourceId, + sourceInstance: instance, + aiDeferred: false, + sourceKeyChecked: true, + limits, + // The import is data; only the summary queued after a fresh insert + // follows the capability (and the person's auto-read setting). + documentAi: () => getAiCapability("documentAi"), + }); + + switch (result.kind) { + case "sourceKey": + return answerHeldKey(system, userId, link, result.match); + case "duplicate": { + // The same bytes were already in the vault (an upload, or another + // archive); the key is now remembered for that document. + const linked = await linkDocument(userId, link, result.document.id); + return respond(system, "duplicate", result.document.id, linked); + } + case "stored": { + const linked = await linkDocument(userId, link, result.document.id); + return respond(system, "imported", result.document.id, linked); + } + case "tooLarge": + return apiError("File is too large.", 413, { + errorCode: "documents.inbound.fileTooLarge", + reason: "fileTooLarge", + maxFileBytes: result.maxFileBytes, + }); + case "empty": + case "unsupportedType": + return apiError("This file type is not supported.", 415, { + errorCode: "documents.inbound.fileType", + reason: "unsupportedType", + }); + case "quotaExceeded": + return apiError("Storage quota exceeded.", 413, { + errorCode: "documents.inbound.quotaExceeded", + reason: "quotaExceeded", + quotaBytes: result.quotaBytes, + usedBytes: result.usedBytes, + }); + case "aliasLimit": + return apiError( + "This file is already stored under too many source ids.", + 409, + { errorCode: "documents.inbound.sourceAliasLimit" }, + ); + case "episodeNotFound": + case "encounterNotFound": + // No pre-links are passed on this path; unreachable, but a result + // the switch does not name must not fall through silently. + return sourceErrorResponse("linkTargetNotFound"); + } +} diff --git a/src/app/api/documents/sources/[system]/route.ts b/src/app/api/documents/sources/[system]/route.ts index 0057e4c24..91cfb0dba 100644 --- a/src/app/api/documents/sources/[system]/route.ts +++ b/src/app/api/documents/sources/[system]/route.ts @@ -6,8 +6,9 @@ * (`DOCUMENT_SOURCE_ORIGINS`), tests the connection live before storing * anything (a wrong token or an unreachable host is reported here, not on the * first search), then stores the token encrypted. A PUT without a token keeps - * the stored one, so the address or the Papra organization can change without - * typing the key again; the first save must carry it. The response is the + * the stored one, so the base path or the Papra organization can change + * without typing the key again, but only on the same origin: a new origin, and + * the first save, must carry it (422 `documents.sources.tokenRequired`). The response is the * connection without its token, like every read of it. * * DELETE removes the row, and with it the token. @@ -100,8 +101,16 @@ export const PUT = apiHandler( } const existing = await loadConnection(userId, system); + // The stored token is only ever sent back to the origin it was saved + // for. A new address on another origin needs the token typed again, or + // saving an address would hand the old archive's token to whatever the + // new one is. + const reusable = + existing !== null && + evaluateSourceBaseUrl(existing.baseUrl, new Set([verdict.origin])).ok; const token = - parsed.data.token ?? (existing ? decrypt(existing.tokenEncrypted) : null); + parsed.data.token ?? + (existing && reusable ? decrypt(existing.tokenEncrypted) : null); if (!token) { return apiError("Enter the API token.", 422, { errorCode: "documents.sources.tokenRequired", @@ -165,9 +174,13 @@ export const PUT = apiHandler( export const DELETE = apiHandler( async (request: NextRequest, { params }: RouteParams) => { const auth = await requireAuth(); - // No list needed to forget a connection: an operator who removed the - // origin must not leave the person unable to delete the stored token. - const refused = await admitDocumentSourceCaller(auth, { needsList: false }); + // Neither the list nor the module is needed to forget a connection: an + // operator who removed the origin, or a person who switched Documents + // off, must still be able to delete the stored token. + const refused = await admitDocumentSourceCaller(auth, { + needsList: false, + needsModule: false, + }); if (refused) return refused; const system = await systemParam(params); if (system instanceof Response) return system; diff --git a/src/app/api/documents/sources/[system]/search/route.ts b/src/app/api/documents/sources/[system]/search/route.ts index 7296048e8..edaf7a9c3 100644 --- a/src/app/api/documents/sources/[system]/search/route.ts +++ b/src/app/api/documents/sources/[system]/search/route.ts @@ -9,6 +9,10 @@ * `new`, `imported` (with the vault document's id) or `deleted` (the person * deleted it here; importing it again is refused). Nothing is stored. * + * When the source ignores a filter and the local re-check empties a page, the + * route reads on (up to five source pages) and answers with the last page it + * read, so the client's "load more" continues after it. + * * Sixty searches a minute, shared with the tag read. Cookie-only and * owner-only; see `admitDocumentSourceCaller`. */ @@ -30,9 +34,10 @@ import { sourceErrorResponse, systemParam, } from "@/lib/documents/sources/route-support"; -import type { - DocumentSourceResultDto, - DocumentSourceSearchDto, +import { + DOCUMENT_PICKER_MAX_PAGE, + type DocumentSourceResultDto, + type DocumentSourceSearchDto, } from "@/lib/documents/sources/types"; import { annotate } from "@/lib/logging/context"; import { rateLimitHeaders } from "@/lib/rate-limit"; @@ -40,6 +45,9 @@ import { documentSourceSearchSchema } from "@/lib/validations/document-sources"; export const dynamic = "force-dynamic"; +/** Source pages one search request may read to find a non-empty one. */ +const SEARCH_PAGES_PER_REQUEST = 5; + type RouteParams = { params: Promise<{ system: string }> }; export const GET = apiHandler( @@ -77,16 +85,34 @@ export const GET = apiHandler( ); } - let found: { items: SourceListItem[]; hasMore: boolean }; + // A page the local tag/date re-check emptied is not an answer: the next + // source page may hold matches. Read on, a bounded number of pages, and + // report the last one read so "load more" continues after it. + let found: { items: SourceListItem[]; hasMore: boolean } = { + items: [], + hasMore: false, + }; + let lastPage = page; + let instance: string; try { const client = await openConnection(userId, system); - found = await client.search({ - q, - tagId: tag ?? null, - from: from ?? null, - to: to ?? null, - page, - }); + instance = client.instance; + for ( + let next = page; + next < page + SEARCH_PAGES_PER_REQUEST && + next <= DOCUMENT_PICKER_MAX_PAGE; + next += 1 + ) { + found = await client.search({ + q, + tagId: tag ?? null, + from: from ?? null, + to: to ?? null, + page: next, + }); + lastPage = next; + if (found.items.length > 0 || !found.hasMore) break; + } } catch (err) { return responseForSourceFailure(err); } @@ -95,6 +121,7 @@ export const GET = apiHandler( userId, system, found.items.map((item) => item.sourceId), + instance, ); const results: DocumentSourceResultDto[] = found.items.map((item) => { const held = states.get(item.sourceId); @@ -116,6 +143,7 @@ export const GET = apiHandler( meta: { system, page, + pagesRead: lastPage - page + 1, count: results.length, filtered: Boolean(q || tag || from || to), }, @@ -123,8 +151,8 @@ export const GET = apiHandler( const body: DocumentSourceSearchDto = { results, - page, - hasMore: found.hasMore, + page: lastPage, + hasMore: found.hasMore && lastPage < DOCUMENT_PICKER_MAX_PAGE, }; return apiSuccess(body); }, diff --git a/src/app/api/documents/sources/route.ts b/src/app/api/documents/sources/route.ts index 0a91b184b..e79f4b22f 100644 --- a/src/app/api/documents/sources/route.ts +++ b/src/app/api/documents/sources/route.ts @@ -8,9 +8,10 @@ * No request leaves the server here. * * Cookie-only and owner-only like every route under this path - * (`admitDocumentSourceCaller`). With the list unset it answers 200 - * `available: false` rather than 404, because this is the one read the client - * makes to decide whether to show the picker at all. + * (`admitDocumentSourceCaller`), but it needs neither the documents module nor + * the list: with the list unset it answers 200 `available: false` (plus any + * rows still stored, so they can be deleted) rather than 404, because this is + * the one read the client makes to decide what to show. */ import { apiHandler, requireAuth } from "@/lib/api-handler"; import { apiSuccess } from "@/lib/api-response"; @@ -30,21 +31,27 @@ export const dynamic = "force-dynamic"; export const GET = apiHandler(async () => { const auth = await requireAuth(); - const refused = await admitDocumentSourceCaller(auth, { needsList: false }); + const refused = await admitDocumentSourceCaller(auth, { + needsList: false, + needsModule: false, + }); if (refused) return refused; annotate({ action: { name: "documents.sources.status" } }); + // Rows are listed even with the picker off: a connection whose origin the + // operator removed (or whose list is gone) must stay visible so it can be + // deleted. `originAllowed` is false for each of them then. + const rows = await listConnections(auth.user.id); if (!documentSourcesAvailable()) { const off: DocumentSourcesStatusDto = { available: false, allowedOrigins: [], - connections: [], + connections: rows.map(toConnectionDto), }; return apiSuccess(off); } - const rows = await listConnections(auth.user.id); const status: DocumentSourcesStatusDto = { available: true, allowedOrigins: listedDocumentSourceOrigins(), diff --git a/src/lib/documents/ingest.ts b/src/lib/documents/ingest.ts index e38df19d3..2e21291c4 100644 --- a/src/lib/documents/ingest.ts +++ b/src/lib/documents/ingest.ts @@ -93,6 +93,11 @@ export interface IngestInput { encounterIds: string[]; sourceSystem: DocumentSourceSystemValue | null; sourceId: string | null; + /** + * The source instance (origin) of the key, or null for a key sent without + * one; see `src/lib/documents/source-key.ts` for how null matches. + */ + sourceInstance: string | null; /** Hold back automatic AI reading (`aiRead=defer`). */ aiDeferred: boolean; /** @@ -151,6 +156,7 @@ async function answerDuplicate( existing.id, input.sourceSystem, input.sourceId, + input.sourceInstance, ); if (remembered === "limit") return { kind: "aliasLimit" }; } @@ -161,6 +167,7 @@ export async function ingestDocument( input: IngestInput, ): Promise { const { userId, sourceSystem, sourceId, limits, bytes } = input; + const sourceInstance = sourceId ? input.sourceInstance : null; // A source key answers before the bytes are looked at: an import re-sending // what it sent before is a duplicate, and one re-sending a document the @@ -168,7 +175,12 @@ export async function ingestDocument( // count (the unique index has no `deleted_at` predicate), and past the purge // the ledger remembers. if (sourceSystem && sourceId && !input.sourceKeyChecked) { - const match = await findSourceKey(userId, sourceSystem, sourceId); + const match = await findSourceKey( + userId, + sourceSystem, + sourceId, + sourceInstance, + ); if (match) return { kind: "sourceKey", match }; } @@ -258,6 +270,7 @@ export async function ingestDocument( : new Date(), sourceSystem, sourceId, + sourceInstance, aiReadDeferred: input.aiDeferred, }, omit: { contentEncrypted: true }, @@ -296,7 +309,12 @@ export async function ingestDocument( // source key, or the same bytes. Surface the winner exactly as the fast // paths above would have. if (sourceSystem && sourceId) { - const match = await findSourceKey(userId, sourceSystem, sourceId); + const match = await findSourceKey( + userId, + sourceSystem, + sourceId, + sourceInstance, + ); if (match) return { kind: "sourceKey", match }; } const winner = await prisma.inboundDocument.findFirst({ diff --git a/src/lib/documents/source-key.ts b/src/lib/documents/source-key.ts index 61fa6580d..f089aa3b4 100644 --- a/src/lib/documents/source-key.ts +++ b/src/lib/documents/source-key.ts @@ -11,8 +11,21 @@ * 3. `DocumentImportKey` — a key whose document the purge has removed. * * The first two resolve to a document (live → duplicate, tombstoned → - * deleted); the third only to "deleted". The upload route and the lookup - * route both ask here, so they cannot disagree about what a key means. + * deleted); the third only to "deleted". The upload route, the lookup route + * and the document picker all ask here, so they cannot disagree about what a + * key means. + * + * v1.39.3 — a key also names its instance (`sourceInstance`, the origin of + * the Paperless-ngx or Papra it came from): id 123 on one Paperless and id + * 123 on another are different documents. A null on either side is a + * wildcard, so nothing stored before the column existed is forgotten: + * + * - a key stored without an instance (a v1.39.2 import, an older script, a + * workflow that does not send one) matches a lookup for any instance of + * its system; + * - a lookup without an instance matches a key stored for any instance. + * + * Where both an exact and a wildcard match exist, the exact one wins. */ import { prisma } from "@/lib/db"; import { checkRateLimit, type RateLimitResult } from "@/lib/rate-limit"; @@ -23,22 +36,44 @@ export type SourceKeyMatch = | { state: "live"; document: SerialisableDocument } | { state: "deleted"; id: string | null }; +/** + * The instance half of a key match: the same instance or a stored key without + * one; any instance when the lookup has none. + */ +export function instanceMatch(sourceInstance: string | null): { + OR?: Array<{ sourceInstance: string | null }>; +} { + return sourceInstance === null + ? {} + : { OR: [{ sourceInstance }, { sourceInstance: null }] }; +} + +/** Exact instance first, then a key stored without one (nulls sort last). */ +const EXACT_FIRST = { sourceInstance: { sort: "asc", nulls: "last" } } as const; + export async function findSourceKey( userId: string, sourceSystem: DocumentSourceSystemValue, sourceId: string, + sourceInstance: string | null = null, ): Promise { + const key = { + userId, + sourceSystem, + sourceId, + ...instanceMatch(sourceInstance), + }; const own = await prisma.inboundDocument.findFirst({ - where: { userId, sourceSystem, sourceId }, + where: key, omit: { contentEncrypted: true }, + orderBy: EXACT_FIRST, }); const viaAlias = own ? null - : await prisma.documentSourceAlias.findUnique({ - where: { - userId_sourceSystem_sourceId: { userId, sourceSystem, sourceId }, - }, + : await prisma.documentSourceAlias.findFirst({ + where: key, select: { document: { omit: { contentEncrypted: true } } }, + orderBy: EXACT_FIRST, }); const document = own ?? viaAlias?.document ?? null; if (document) { @@ -46,10 +81,8 @@ export async function findSourceKey( ? { state: "deleted", id: document.id } : { state: "live", document }; } - const purged = await prisma.documentImportKey.findUnique({ - where: { - userId_sourceSystem_sourceId: { userId, sourceSystem, sourceId }, - }, + const purged = await prisma.documentImportKey.findFirst({ + where: key, select: { id: true }, }); return purged ? { state: "deleted", id: null } : null; @@ -74,13 +107,14 @@ export async function rememberSourceAlias( documentId: string, sourceSystem: DocumentSourceSystemValue, sourceId: string, + sourceInstance: string | null = null, ): Promise<"remembered" | "known" | "limit"> { const held = await prisma.documentSourceAlias.count({ where: { userId, documentId }, }); if (held >= MAX_SOURCE_ALIASES_PER_DOCUMENT) return "limit"; const { count } = await prisma.documentSourceAlias.createMany({ - data: [{ userId, documentId, sourceSystem, sourceId }], + data: [{ userId, documentId, sourceSystem, sourceInstance, sourceId }], skipDuplicates: true, }); return count > 0 ? "remembered" : "known"; @@ -123,33 +157,56 @@ export async function findSourceKeyStates( userId: string, sourceSystem: DocumentSourceSystemValue, sourceIds: string[], + sourceInstance: string | null = null, ): Promise> { const states = new Map(); const ids = [...new Set(sourceIds)]; if (ids.length === 0) return states; + const where = { + userId, + sourceSystem, + sourceId: { in: ids }, + ...instanceMatch(sourceInstance), + }; const [own, aliases, purged] = await Promise.all([ prisma.inboundDocument.findMany({ - where: { userId, sourceSystem, sourceId: { in: ids } }, - select: { id: true, sourceId: true, deletedAt: true }, + where, + select: { + id: true, + sourceId: true, + sourceInstance: true, + deletedAt: true, + }, }), prisma.documentSourceAlias.findMany({ - where: { userId, sourceSystem, sourceId: { in: ids } }, + where, select: { sourceId: true, + sourceInstance: true, document: { select: { id: true, deletedAt: true } }, }, }), prisma.documentImportKey.findMany({ - where: { userId, sourceSystem, sourceId: { in: ids } }, + where, select: { sourceId: true }, }), ]); + // A wildcard (instance-less) row first, so an exact-instance row after it + // wins, as it does in `findSourceKey`. + const wildcardFirst = ( + rows: T[], + ) => + [...rows].sort( + (a, b) => + Number(a.sourceInstance === sourceInstance) - + Number(b.sourceInstance === sourceInstance), + ); // Lowest precedence first, so the document's own key wins, as it does in // `findSourceKey`. for (const row of purged) states.set(row.sourceId, { state: "deleted" }); - for (const row of aliases) { + for (const row of wildcardFirst(aliases)) { states.set( row.sourceId, row.document.deletedAt @@ -157,7 +214,7 @@ export async function findSourceKeyStates( : { state: "imported", documentId: row.document.id }, ); } - for (const row of own) { + for (const row of wildcardFirst(own)) { if (!row.sourceId) continue; states.set( row.sourceId, diff --git a/src/lib/documents/sources/__tests__/clients.test.ts b/src/lib/documents/sources/__tests__/clients.test.ts index 5bc780029..78f4f536b 100644 --- a/src/lib/documents/sources/__tests__/clients.test.ts +++ b/src/lib/documents/sources/__tests__/clients.test.ts @@ -171,8 +171,10 @@ describe("Paperless-ngx", () => { const list = calledUrls().find((u) => u.pathname.endsWith("/documents/")); expect(list?.searchParams.get("title__icontains")).toBe("blood"); expect(list?.searchParams.get("tags__id__all")).toBe("7"); - expect(list?.searchParams.get("created__date__gte")).toBe("2025-01-01"); - expect(list?.searchParams.get("created__date__lte")).toBe("2025-12-31"); + expect(list?.searchParams.get("created__gte")).toBe("2025-01-01"); + expect(list?.searchParams.get("created__lte")).toBe("2025-12-31"); + // API version 9 made `created` a date; the `__date` lookups are gone. + expect(list?.searchParams.has("created__date__gte")).toBe(false); expect(list?.searchParams.get("page")).toBe("2"); expect(list?.searchParams.get("page_size")).toBe("25"); @@ -311,6 +313,76 @@ describe("Paperless-ngx", () => { }); }); +describe("ids never shape a request path", () => { + it("refuses a Paperless id that is not a number, and dot segments, before dialling", async () => { + const client = sourceClient(PAPERLESS); + for (const id of ["..", ".", "41/../x", "abc", "41?x=1"]) { + const err = await failure(client.document(id)); + expect((err as DocumentSourceError).code, id).toBe("notFound"); + expect( + ((await failure(client.download(id, 10))) as DocumentSourceError).code, + ).toBe("notFound"); + } + expect(safeFetch).not.toHaveBeenCalled(); + }); + + it("refuses a Papra id or organization id outside [A-Za-z0-9_-]", async () => { + for (const id of ["..", "doc/1", "doc%2F1", "doc 1"]) { + await failure(sourceClient(PAPRA).document(id)); + } + expect(() => + sourceClient({ ...PAPRA, organizationId: "../admin" }), + ).toThrow(DocumentSourceError); + expect(safeFetch).not.toHaveBeenCalled(); + }); + + it("leaves a result with an unusable id out of the list", async () => { + serve({ + "/paperless/api/tags/": () => json({ results: [], next: null }), + "/paperless/api/documents/": () => + json({ + next: null, + results: [ + { id: "..", title: "Odd", created: "2025-01-01", tags: [] }, + { id: 5, title: "Fine", created: "2025-01-01", tags: [] }, + ], + }), + }); + const result = await sourceClient(PAPERLESS).search({ + q: "", + tagId: null, + from: null, + to: null, + page: 1, + }); + expect(result.items.map((i) => i.sourceId)).toEqual(["5"]); + }); +}); + +describe("Paperless-ngx tags", () => { + it("pages through tags beyond the first 250", async () => { + serve({ + "/paperless/api/tags/": (url) => { + const page = Number(url.searchParams.get("page")); + const results = Array.from( + { length: page === 1 ? 250 : 3 }, + (_, i) => ({ + id: page * 1000 + i, + name: `t${page}-${i}`, + }), + ); + return json({ results, next: page === 1 ? "more" : null }); + }, + }); + const tags = await sourceClient(PAPERLESS).tags(); + expect(tags).toHaveLength(253); + expect(calledUrls().map((u) => u.searchParams.get("page"))).toEqual([ + "1", + "2", + ]); + }); +}); + describe("Papra", () => { const tags = { tags: [ diff --git a/src/lib/documents/sources/clients.ts b/src/lib/documents/sources/clients.ts index 7bca6ed92..a5d1cfa85 100644 --- a/src/lib/documents/sources/clients.ts +++ b/src/lib/documents/sources/clients.ts @@ -29,6 +29,8 @@ import { import { DOCUMENT_PICKER_MAX_TAGS, DOCUMENT_PICKER_PAGE_SIZE, + isSourceDocumentId, + PAPRA_ID, type DocumentPickerSystem, type DocumentSourceTagDto, } from "./types"; @@ -64,6 +66,11 @@ export interface SourceDocumentMeta { export interface SourceClient { readonly system: DocumentPickerSystem; + /** + * The instance this client talks to, as the normalised origin every source + * key it produces carries (`sourceInstance`). + */ + readonly instance: string; test(): Promise; tags(): Promise; search( @@ -84,6 +91,16 @@ export interface SourceConnection { token: string; } +/** Paperless tags are read this many per page. */ +const TAG_PAGE_SIZE = 250; + +/** Refuse an id that is not one the system issues, before it reaches a URL. */ +function requireId(system: DocumentPickerSystem, id: string): void { + if (!isSourceDocumentId(system, id)) { + throw new DocumentSourceError("notFound"); + } +} + // ── Defensive readers ────────────────────────────────────────────────────── type Json = Record; @@ -181,26 +198,36 @@ function paperless(connection: SourceConnection): SourceClient { } async function listTags(): Promise { - const query = new URLSearchParams({ - page_size: String(DOCUMENT_PICKER_MAX_TAGS), - ordering: "name", - }); - const { body, response } = await sourceJson(target, `/api/tags/?${query}`); - checkVersion(response); - const page = asRecord(body); - if (!page) throw shapeError(); + // Paged by number rather than by following `next`: the link is the + // server's to write, the page number is ours. const tags: DocumentSourceTagDto[] = []; - for (const raw of asArray(page.results)) { - const tag = asRecord(raw); - const id = asText(tag?.id); - const name = asText(tag?.name); - if (id && name) tags.push({ id, name }); + for (let pageNo = 1; tags.length < DOCUMENT_PICKER_MAX_TAGS; pageNo += 1) { + const query = new URLSearchParams({ + page: String(pageNo), + page_size: String(TAG_PAGE_SIZE), + ordering: "name", + }); + const { body, response } = await sourceJson( + target, + `/api/tags/?${query}`, + ); + checkVersion(response); + const page = asRecord(body); + if (!page) throw shapeError(); + for (const raw of asArray(page.results)) { + const tag = asRecord(raw); + const id = asText(tag?.id); + const name = asText(tag?.name); + if (id && name) tags.push({ id, name }); + } + if (typeof page.next !== "string" || !page.next) break; } - return tags; + return tags.slice(0, DOCUMENT_PICKER_MAX_TAGS); } return { system: "PAPERLESS", + instance: connection.origin, async test() { const { body, response } = await sourceJson( @@ -223,8 +250,10 @@ function paperless(connection: SourceConnection): SourceClient { }); if (params.q) query.set("title__icontains", params.q); if (params.tagId) query.set("tags__id__all", params.tagId); - if (params.from) query.set("created__date__gte", params.from); - if (params.to) query.set("created__date__lte", params.to); + // API version 9 made `created` a plain date, so the range filters on + // it directly (`created__gte` / `created__lte`), both ends inclusive. + if (params.from) query.set("created__gte", params.from); + if (params.to) query.set("created__lte", params.to); let page: Json | null; let names: Map; @@ -250,7 +279,7 @@ function paperless(connection: SourceConnection): SourceClient { for (const raw of asArray(page.results)) { const doc = asRecord(raw); const id = asText(doc?.id); - if (!doc || !id) continue; + if (!doc || !id || !isSourceDocumentId("PAPERLESS", id)) continue; const tagIds = asArray(doc.tags) .map(asText) .filter((t): t is string => t !== null); @@ -275,6 +304,7 @@ function paperless(connection: SourceConnection): SourceClient { }, async document(sourceId) { + requireId("PAPERLESS", sourceId); const { body, response } = await sourceJson( target, `/api/documents/${encodeURIComponent(sourceId)}/?fields=id,title,created,added,original_file_name`, @@ -290,7 +320,8 @@ function paperless(connection: SourceConnection): SourceClient { }; }, - download(sourceId, maxFileBytes) { + async download(sourceId, maxFileBytes) { + requireId("PAPERLESS", sourceId); // The original, not the archived PDF/A: the same bytes a Paperless // workflow or the import script sends, so they dedup against each other. return sourceDownload( @@ -310,7 +341,7 @@ function papraTagTerm(name: string): string { } function papra(connection: SourceConnection): SourceClient { - if (!connection.organizationId) { + if (!connection.organizationId || !PAPRA_ID.test(connection.organizationId)) { throw new DocumentSourceError("notFound"); } const org = `/api/organizations/${encodeURIComponent(connection.organizationId)}`; @@ -345,6 +376,7 @@ function papra(connection: SourceConnection): SourceClient { return { system: "PAPRA", + instance: connection.origin, async test() { const { body } = await sourceJson( @@ -381,7 +413,7 @@ function papra(connection: SourceConnection): SourceClient { for (const raw of documents) { const doc = asRecord(raw); const id = asText(doc?.id); - if (!doc || !id) continue; + if (!doc || !id || !isSourceDocumentId("PAPRA", id)) continue; const tags = asArray(doc.tags) .map(asRecord) .filter((t): t is Json => t !== null); @@ -417,6 +449,7 @@ function papra(connection: SourceConnection): SourceClient { }, async document(sourceId) { + requireId("PAPRA", sourceId); const { body } = await sourceJson( target, `${org}/documents/${encodeURIComponent(sourceId)}`, @@ -435,7 +468,8 @@ function papra(connection: SourceConnection): SourceClient { }; }, - download(sourceId, maxFileBytes) { + async download(sourceId, maxFileBytes) { + requireId("PAPRA", sourceId); return sourceDownload( download, `${org}/documents/${encodeURIComponent(sourceId)}/file`, diff --git a/src/lib/documents/sources/route-support.ts b/src/lib/documents/sources/route-support.ts index 295944b8c..a1b9aba86 100644 --- a/src/lib/documents/sources/route-support.ts +++ b/src/lib/documents/sources/route-support.ts @@ -39,12 +39,13 @@ import { * Each route calls `requireAuth()` itself, in its own module, so the sharing * guard's scan sees every one of them as a bare, owner-only route rather than * a helper it cannot follow. This adds the rest: a cookie transport, the - * documents module, and (unless `needsList` is false, which only the status - * read passes) the operator's list. + * documents module and the operator's list. The status read and the delete + * pass `needsList: false, needsModule: false`: they touch only the person's + * own row and never the network. */ export async function admitDocumentSourceCaller( auth: AuthContext, - options: { needsList?: boolean } = {}, + options: { needsList?: boolean; needsModule?: boolean } = {}, ): Promise { if (auth.authMethod !== "cookie") { return apiError( @@ -53,8 +54,13 @@ export async function admitDocumentSourceCaller( { errorCode: "documents.sources.browserOnly" }, ); } - const gate = await requireModuleEnabled(auth.user.id, "inboundDocuments"); - if (!gate.enabled) return gate.response; + // Reading and removing one's own connection needs neither the module nor + // the list: a person who switched Documents off, or whose operator removed + // the list, must still be able to see and delete the stored token. + if (options.needsModule !== false) { + const gate = await requireModuleEnabled(auth.user.id, "inboundDocuments"); + if (!gate.enabled) return gate.response; + } if (options.needsList !== false && !documentSourcesAvailable()) { return sourceErrorResponse("unavailable"); } diff --git a/src/lib/documents/sources/types.ts b/src/lib/documents/sources/types.ts index 1e6cdf774..6dd8ddf35 100644 --- a/src/lib/documents/sources/types.ts +++ b/src/lib/documents/sources/types.ts @@ -38,8 +38,26 @@ export const DOCUMENT_PICKER_MAX_PAGE = 500; export const DOCUMENT_PICKER_QUERY_MAX = 200; /** Documents the sheet imports in one run. */ export const DOCUMENT_PICKER_MAX_SELECTION = 25; -/** Tags offered in the filter. */ -export const DOCUMENT_PICKER_MAX_TAGS = 250; +/** Tags offered in the filter (read in pages of 250). */ +export const DOCUMENT_PICKER_MAX_TAGS = 1000; + +/** Paperless-ngx document ids are integers. */ +export const PAPERLESS_ID = /^\d{1,18}$/; +/** Papra document and organization ids: letters, digits, `_` and `-`. */ +export const PAPRA_ID = /^[A-Za-z0-9_-]{1,128}$/; + +/** + * Is `id` a document id `system` could have issued? Checked before an id is + * put into a request path, so nothing a request carries can add a path + * segment, a query or a `..` to the URL built from it. + */ +export function isSourceDocumentId( + system: DocumentPickerSystem, + id: string, +): boolean { + if (id === "." || id === "..") return false; + return (system === "PAPERLESS" ? PAPERLESS_ID : PAPRA_ID).test(id); +} /** A connection as every route may show it: never the token. */ export interface DocumentSourceConnectionDto { diff --git a/src/lib/export/records-backup.ts b/src/lib/export/records-backup.ts index d924c8c59..2bb4cd278 100644 --- a/src/lib/export/records-backup.ts +++ b/src/lib/export/records-backup.ts @@ -177,6 +177,8 @@ export interface DocumentBackupEntry { */ sourceSystem?: string | null; sourceId?: string | null; + /** v1.39.3 — the source instance (origin); absent in older files. */ + sourceInstance?: string | null; /** Canonical DR only: the import's hold on automatic AI reading. */ aiReadDeferred?: boolean; } @@ -309,6 +311,7 @@ export async function buildRecordsBackupSection( lastIndexOutcome: true, sourceSystem: true, sourceId: true, + sourceInstance: true, aiReadDeferred: true, createdAt: true, updatedAt: true, @@ -330,6 +333,7 @@ export async function buildRecordsBackupSection( summaryEncrypted: true, sourceSystem: true, sourceId: true, + sourceInstance: true, createdAt: true, }, }); @@ -545,6 +549,7 @@ export async function buildRecordsBackupSection( : null, sourceSystem: d.sourceSystem, sourceId: d.sourceId, + sourceInstance: d.sourceInstance, createdAt: d.createdAt.toISOString(), }; diff --git a/src/lib/export/restore-backup.ts b/src/lib/export/restore-backup.ts index 3e60c6b51..d10100e78 100644 --- a/src/lib/export/restore-backup.ts +++ b/src/lib/export/restore-backup.ts @@ -30,6 +30,7 @@ import { Buffer } from "node:buffer"; import { prisma, toJson } from "@/lib/db"; +import { normaliseSourceInstance } from "@/lib/validations/inbound-documents"; import { auditLog } from "@/lib/auth/audit"; import { isStoredBackupReadError, @@ -2057,6 +2058,14 @@ export async function restoreBackup( sourceId: document.sourceSystem ? (document.sourceId ?? null) : null, + // A file written before v1.39.3 has no instance: the key then + // matches any instance of its system, as it did when stored. + sourceInstance: + document.sourceSystem && document.sourceId + ? document.sourceInstance + ? normaliseSourceInstance(document.sourceInstance) + : null + : null, createdAt: new Date(document.createdAt!), updatedAt: new Date(document.updatedAt!), })), diff --git a/src/lib/jobs/document-purge.ts b/src/lib/jobs/document-purge.ts index 40106a370..d98e9a988 100644 --- a/src/lib/jobs/document-purge.ts +++ b/src/lib/jobs/document-purge.ts @@ -58,9 +58,12 @@ export async function purgeExpiredDocumentTombstones( userId: true, sourceSystem: true, sourceId: true, + sourceInstance: true, // Further keys the import answered with this document (same bytes). // They cascade away with the row, so they are copied here first. - sourceAliases: { select: { sourceSystem: true, sourceId: true } }, + sourceAliases: { + select: { sourceSystem: true, sourceId: true, sourceInstance: true }, + }, }, }); const keys = keyed.flatMap((row) => [ @@ -70,6 +73,7 @@ export async function purgeExpiredDocumentTombstones( userId: row.userId, sourceSystem: row.sourceSystem, sourceId: row.sourceId, + sourceInstance: row.sourceInstance, }, ] : []), @@ -77,6 +81,7 @@ export async function purgeExpiredDocumentTombstones( userId: row.userId, sourceSystem: alias.sourceSystem, sourceId: alias.sourceId, + sourceInstance: alias.sourceInstance, })), ]); if (keys.length > 0) { diff --git a/src/lib/validations/backup.ts b/src/lib/validations/backup.ts index d9aa93bfe..c42b5b566 100644 --- a/src/lib/validations/backup.ts +++ b/src/lib/validations/backup.ts @@ -83,6 +83,7 @@ import { } from "@/lib/validations/emergency-profile"; import { DOCUMENT_SOURCE_ID_MAX, + DOCUMENT_SOURCE_INSTANCE_MAX, DOCUMENT_SOURCE_SYSTEMS, } from "@/lib/validations/inbound-documents"; import { REMINDER_EVENT_SOURCES } from "@/lib/measurement-reminders/satisfy"; @@ -1636,6 +1637,14 @@ const documentBackupSchema = z // restore cannot write a system the detail sheet has no name for. sourceSystem: z.enum(DOCUMENT_SOURCE_SYSTEMS).nullable().optional(), sourceId: z.string().max(DOCUMENT_SOURCE_ID_MAX).nullable().optional(), + // v1.39.3 — the source instance (an origin). Length-bounded here and + // normalised on restore; a value that is not an origin restores as null + // (matches any instance) rather than failing the restore. + sourceInstance: z + .string() + .max(DOCUMENT_SOURCE_INSTANCE_MAX) + .nullable() + .optional(), aiReadDeferred: z.boolean().optional(), summary: z.string().nullable().optional(), createdAt: isoDateTime.optional(), diff --git a/src/lib/validations/document-sources.ts b/src/lib/validations/document-sources.ts index 6997cb97d..d175c601c 100644 --- a/src/lib/validations/document-sources.ts +++ b/src/lib/validations/document-sources.ts @@ -10,6 +10,7 @@ import { DOCUMENT_PICKER_LINK_KINDS, DOCUMENT_PICKER_MAX_PAGE, DOCUMENT_PICKER_QUERY_MAX, + PAPRA_ID, } from "@/lib/documents/sources/types"; import { documentSourceIdSchema, @@ -37,9 +38,7 @@ export const documentSourceSaveSchema = z.object({ organizationId: z .string() .trim() - .min(1) - .max(128) - .regex(printable, "Expected printable characters without spaces") + .regex(PAPRA_ID, "Expected letters, digits, _ or -") .optional(), token: z .string() diff --git a/src/lib/validations/inbound-documents.ts b/src/lib/validations/inbound-documents.ts index ebc8aa030..afe29cd78 100644 --- a/src/lib/validations/inbound-documents.ts +++ b/src/lib/validations/inbound-documents.ts @@ -706,7 +706,41 @@ export const documentSourceIdSchema = z .trim() .min(1) .max(DOCUMENT_SOURCE_ID_MAX) - .regex(/^[\x21-\x7e]+$/u, "Expected printable characters without spaces"); + .regex(/^[\x21-\x7e]+$/u, "Expected printable characters without spaces") + // A path segment of its own name: an importer that builds a URL from the id + // must never be handed one that walks up the source's path. + .refine((v) => v !== "." && v !== "..", "Not a document id"); + +/** Longest source instance (an origin) a key may carry. */ +export const DOCUMENT_SOURCE_INSTANCE_MAX = 512; + +/** + * v1.39.3 — the instance of the source system a key belongs to, as the + * normalised origin (`scheme://host[:port]`) of the Paperless-ngx or Papra it + * came from, or null when the value is not one. A path, query or credentials + * in the value are dropped: two addresses on one origin are one instance. + */ +export function normaliseSourceInstance(value: string): string | null { + try { + const url = new URL(value.trim()); + if (url.protocol !== "http:" && url.protocol !== "https:") return null; + if (url.username || url.password) return null; + return url.origin; + } catch { + return null; + } +} + +/** The instance as a request field: validated, then normalised to its origin. */ +export const documentSourceInstanceSchema = z + .string() + .trim() + .min(1) + .max(DOCUMENT_SOURCE_INSTANCE_MAX) + .refine((v) => normaliseSourceInstance(v) !== null, { + message: "Expected the http(s) address of the source instance", + }) + .transform((v) => normaliseSourceInstance(v) as string); /** * A source key on its own — the query-string form the upload accepts ahead of @@ -715,6 +749,8 @@ export const documentSourceIdSchema = z export const documentSourceKeySchema = z.object({ sourceSystem: z.enum(DOCUMENT_SOURCE_SYSTEMS), sourceId: documentSourceIdSchema, + /** Optional: without it the key matches any instance of the system. */ + sourceInstance: documentSourceInstanceSchema.optional(), }); export const documentCreateSchema = z @@ -731,6 +767,11 @@ export const documentCreateSchema = z * not prose, and it is echoed on the detail sheet. */ sourceId: documentSourceIdSchema.optional(), + /** + * v1.39.3 — which instance of `sourceSystem` (its origin). Optional; a + * key without one matches any instance. + */ + sourceInstance: documentSourceInstanceSchema.optional(), /** * `defer` holds back automatic AI reading for this upload: the thumbnail * and a local text index still run, the summary and the lab staging do @@ -741,6 +782,10 @@ export const documentCreateSchema = z .refine((v) => v.sourceId === undefined || v.sourceSystem !== undefined, { path: ["sourceSystem"], message: "sourceId needs a sourceSystem", + }) + .refine((v) => v.sourceInstance === undefined || v.sourceId !== undefined, { + path: ["sourceInstance"], + message: "sourceInstance needs a sourceId", }); export type DocumentCreateInput = z.infer; diff --git a/tests/integration/document-sources.test.ts b/tests/integration/document-sources.test.ts index e729e7aed..0bcb842e6 100644 --- a/tests/integration/document-sources.test.ts +++ b/tests/integration/document-sources.test.ts @@ -56,6 +56,8 @@ vi.mock("@/lib/db-compat", () => ({ // ── The fake Paperless-ngx behind `safeFetch` ────────────────────────────── const ORIGIN = "http://paperless.lan:8000"; +/** A second Paperless-ngx the operator also listed; same ids, other documents. */ +const ORIGIN_B = "https://paperless-b.example.com"; const TOKEN = "paperless-token-0123456789-secret"; const PDF = (tag: string) => @@ -83,6 +85,10 @@ const DOCS = [ const fake = vi.hoisted(() => ({ calls: [] as Array<{ url: string; opts: Record }>, acceptToken: "", + /** Delay a download so two imports of one key overlap. */ + downloadDelayMs: 0, + /** Answer downloads with this status instead of the file. */ + downloadStatus: 200, })); vi.mock("@/lib/safe-fetch", async (importOriginal) => { @@ -113,6 +119,31 @@ vi.mock("@/lib/safe-fetch", async (importOriginal) => { return json({ results: [{ id: 7, name: "Health" }], next: null }); } if (path === "/api/documents/") { + // Tag 99: a server that ignores the tag filter, with the only match + // on its second page. + if (url.searchParams.get("tags__id__all") === "99") { + const page2 = url.searchParams.get("page") === "2"; + return json({ + next: page2 ? null : "next", + results: page2 + ? [ + { + id: 41, + title: "Blood test", + created: "2024-03-04", + tags: [99], + }, + ] + : [ + { + id: 42, + title: "Discharge letter", + created: "2023-11-20", + tags: [], + }, + ], + }); + } const q = ( url.searchParams.get("title__icontains") ?? "" ).toLowerCase(); @@ -126,8 +157,19 @@ vi.mock("@/lib/safe-fetch", async (importOriginal) => { const download = /^\/api\/documents\/(\d+)\/download\/$/.exec(path); if (download) { const doc = DOCS.find((d) => String(d.id) === download[1]); + if (fake.downloadDelayMs > 0) { + await new Promise((r) => setTimeout(r, fake.downloadDelayMs)); + } + if (fake.downloadStatus !== 200) { + return new Response("", { status: fake.downloadStatus }); + } + // Each instance holds different bytes under the same id. return doc - ? new Response(new Uint8Array(doc.bytes)) + ? new Response( + new Uint8Array( + Buffer.concat([doc.bytes, Buffer.from(`% ${url.origin}\n`)]), + ), + ) : new Response("", { status: 404 }); } const detail = /^\/api\/documents\/(\d+)\/$/.exec(path); @@ -259,7 +301,9 @@ beforeEach(async () => { headerJar.clear(); fake.calls.length = 0; fake.acceptToken = TOKEN; - process.env.DOCUMENT_SOURCE_ORIGINS = ORIGIN; + fake.downloadDelayMs = 0; + fake.downloadStatus = 200; + process.env.DOCUMENT_SOURCE_ORIGINS = `${ORIGIN},${ORIGIN_B}`; }); afterEach(() => { @@ -338,7 +382,7 @@ describe("saving a connection", () => { const read = await (await status()).text(); expect(read).not.toContain(TOKEN); - expect(JSON.parse(read).data.allowedOrigins).toEqual([ORIGIN]); + expect(JSON.parse(read).data.allowedOrigins).toEqual([ORIGIN, ORIGIN_B]); // A save without a token keeps the stored one. const again = await save({ baseUrl: ORIGIN }); @@ -495,7 +539,15 @@ describe("search and import", () => { // The same bytes, uploaded through the vault without a source key. const form = new FormData(); - form.append("file", new Blob([new Uint8Array(DOCS[1].bytes)]), "x.pdf"); + form.append( + "file", + new Blob([ + new Uint8Array( + Buffer.concat([DOCS[1].bytes, Buffer.from(`% ${ORIGIN}\n`)]), + ), + ]), + "x.pdf", + ); const { POST } = await import("@/app/api/documents/inbound/route"); const uploaded = await ( POST as unknown as (r: Request) => Promise @@ -520,6 +572,7 @@ describe("search and import", () => { where: { documentId: uploadedId, sourceSystem: "PAPERLESS", + sourceInstance: ORIGIN, sourceId: "42", }, }), @@ -583,3 +636,139 @@ describe("search and import", () => { expect((await disconnect()).status).toBe(200); }); }); + +describe("review follow-ups", () => { + it("never sends the stored token to a new origin", async () => { + await seedOwner(); + await connect(); + fake.calls.length = 0; + const moved = await save({ baseUrl: ORIGIN_B }); + expect(moved.status).toBe(422); + expect((await moved.json()).meta.errorCode).toBe( + "documents.sources.tokenRequired", + ); + expect(fake.calls).toHaveLength(0); + // A new path on the same origin keeps the token. + const samePlace = await save({ baseUrl: `${ORIGIN}/` }); + expect(samePlace.status).toBe(200); + expect(fake.calls.every((c) => c.url.startsWith(ORIGIN))).toBe(true); + }); + + it("keeps the same id on two instances apart, and lets a key without an instance match any", async () => { + await seedOwner(); + await connect(); + const prisma = getPrismaClient(); + + const first = await importDoc({ sourceId: "41" }); + expect(first.status).toBe(201); + const a = (await first.json()).data.documentId as string; + expect( + (await prisma.inboundDocument.findUniqueOrThrow({ where: { id: a } })) + .sourceInstance, + ).toBe(ORIGIN); + + // Switch the connection to the other Paperless-ngx. + expect((await save({ baseUrl: ORIGIN_B, token: TOKEN })).status).toBe(200); + const listed = (await (await search("q=blood")).json()).data.results[0]; + expect(listed).toMatchObject({ sourceId: "41", state: "new" }); + const second = await importDoc({ sourceId: "41" }); + expect(second.status).toBe(201); + const b = (await second.json()).data.documentId as string; + expect(b).not.toBe(a); + expect( + (await prisma.inboundDocument.findUniqueOrThrow({ where: { id: b } })) + .sourceInstance, + ).toBe(ORIGIN_B); + + // A v1.39.2 import of #42 carries no instance: it answers for any + // instance of its system, so the picker neither re-imports it nor + // forgets a deletion of it. + await prisma.inboundDocument.create({ + data: { + userId, + kind: "OTHER", + mimeType: "application/pdf", + byteSize: 1, + contentEncrypted: Buffer.from("x"), + contentSha256: "legacy-sha", + status: "STORED", + sourceSystem: "PAPERLESS", + sourceId: "42", + deletedAt: new Date(), + }, + }); + const legacy = (await (await search("q=discharge")).json()).data.results[0]; + expect(legacy).toMatchObject({ sourceId: "42", state: "deleted" }); + const refused = await importDoc({ sourceId: "42" }); + expect((await refused.json()).data.outcome).toBe("deleted"); + }); + + it("refuses an id that could leave the document path, before dialling", async () => { + await seedOwner(); + await connect(); + fake.calls.length = 0; + for (const sourceId of ["..", ".", "41/../../admin", "abc", "41?x=1"]) { + const res = await importDoc({ sourceId }); + expect(res.status, sourceId).toBe(422); + } + expect(fake.calls).toHaveLength(0); + }); + + it("downloads a key once when two imports of it overlap", async () => { + await seedOwner(); + await connect(); + fake.downloadDelayMs = 300; + const [one, two] = await Promise.all([ + importDoc({ sourceId: "41" }), + importDoc({ sourceId: "41" }), + ]); + const outcomes = [ + (await one.json()).data.outcome, + (await two.json()).data.outcome, + ].sort(); + expect(outcomes).toEqual(["duplicate", "imported"]); + expect(downloads()).toBe(1); + expect(await getPrismaClient().inboundDocument.count()).toBe(1); + }); + + it("does not charge the upload allowance for a fetch that failed", async () => { + await seedOwner(); + await connect(); + fake.downloadStatus = 500; + const res = await importDoc({ sourceId: "41" }); + expect(res.status).toBe(502); + expect( + await getPrismaClient().rateLimit.count({ + where: { key: `documents-upload:${userId}` }, + }), + ).toBe(0); + }); + + it("reads past a page the local re-check emptied", async () => { + await seedOwner(); + await connect(); + const res = await search("tag=99"); + expect(res.status).toBe(200); + const data = (await res.json()).data; + expect(data.page).toBe(2); + expect(data.results.map((r: { sourceId: string }) => r.sourceId)).toEqual([ + "41", + ]); + }); + + it("shows and deletes a connection with the module off and the list gone", async () => { + await seedOwner(); + await connect(); + delete process.env.DOCUMENT_SOURCE_ORIGINS; + await getPrismaClient().user.update({ + where: { id: userId }, + data: { modulePreferencesJson: { inboundDocuments: false } }, + }); + const read = (await (await status()).json()).data; + expect(read.available).toBe(false); + expect(read.connections).toHaveLength(1); + expect(read.connections[0].originAllowed).toBe(false); + expect((await disconnect()).status).toBe(200); + expect(await getPrismaClient().documentSourceConnection.count()).toBe(0); + }); +});