Merge branch 'feat/document-picker' into release/v1.39.3

This commit is contained in:
Marc-André Bombeck committed 2026-09-26 17:00:45 +02:00
commit a8cf4c1dad
71 files changed
+8934 -375

No files matched your search

@@ -579,6 +579,13 @@ const REGISTRY: DestructiveEntry[] = [
recovery: "permanent",
confirm: ["AlertDialog"],
})),
{
file: "components/settings/integrations/document-sources-card.tsx",
destroys:
"the stored Paperless-ngx or Papra address and API token; documents already imported stay in the vault — reconnecting means typing the token again",
recovery: "permanent",
confirm: ["AlertDialog"],
},
];
// ── Detectors ─────────────────────────────────────────────────────────────
@@ -0,0 +1,106 @@
/**
* The document picker's routes (#1038) are owner-only, cookie-only, and stay
* that way.
*
* A connection is the owner's credential to a system outside HealthLog. A
* delegate or guardian driving it would read the owner's whole Paperless-ngx or
* Papra archive, which no grant covers; a Bearer (the native client, a script)
* has no business managing it either. The rule is in two halves, and this file
* freezes both at every route module under `src/app/api/documents/sources`:
*
* 1. the module calls a bare `requireAuth()` itself (no scope, so a narrow
* token is refused, and a request acting on another record is refused
* 403 `sharing.not_permitted`), and names none of the resolvers that
* would admit a delegate, a guardian, or an actor surface;
* 2. every handler passes the result through `admitDocumentSourceCaller`,
* which refuses any transport but the cookie.
*
* The scan asserts it found the modules it expects, so an empty directory or a
* moved tree fails rather than passing on nothing.
*
* Mutation checks: swap one route's `requireAuth()` for
* `requireRecordAuth("write", "documents")`, or drop its
* `admitDocumentSourceCaller` call, and this file fails naming that module.
*/
import { readFileSync, readdirSync } from "node:fs";
import { join, relative } from "node:path";
import { describe, expect, it } from "vitest";
import { stripComments } from "./helpers/source-files";
const ROOT = join(process.cwd(), "src/app/api/documents/sources");
function routeModules(): string[] {
const out: string[] = [];
const walk = (dir: string) => {
for (const entry of readdirSync(dir, { withFileTypes: true })) {
const full = join(dir, entry.name);
if (entry.isDirectory()) walk(full);
else if (entry.name === "route.ts") out.push(full);
}
};
walk(ROOT);
return out.map((file) => relative(ROOT, file)).sort();
}
const EXPECTED = [
"[system]/import/route.ts",
"[system]/route.ts",
"[system]/search/route.ts",
"[system]/tags/route.ts",
"[system]/test/route.ts",
"route.ts",
];
describe("document-source routes are owner-only and cookie-only", () => {
const modules = routeModules();
it("finds exactly the route modules it freezes", () => {
expect(modules).toEqual(EXPECTED);
});
it.each(EXPECTED)(
"%s resolves a bare requireAuth and no wider resolver",
(rel) => {
const source = stripComments(readFileSync(join(ROOT, rel), "utf8"));
expect(source).toMatch(/requireAuth\(\s*\)/);
expect(source).not.toMatch(/requireAuth\(\s*[^)\s]/);
for (const wider of [
"requireRecordAuth",
"requireGuardianAuth",
"requireActorAuth",
"requireCookieAuth",
"requireBearerAuth",
]) {
expect(source, `${rel} names ${wider}`).not.toContain(wider);
}
},
);
it.each(EXPECTED)(
"%s admits every handler through the cookie check",
(rel) => {
const source = stripComments(readFileSync(join(ROOT, rel), "utf8"));
const handlers = source.match(
/export const (GET|POST|PUT|PATCH|DELETE)\b/g,
);
expect(handlers?.length ?? 0).toBeGreaterThan(0);
const auths = source.match(/await requireAuth\(\s*\)/g) ?? [];
const admits = source.match(/await admitDocumentSourceCaller\(/g) ?? [];
expect(auths.length).toBe(handlers!.length);
expect(admits.length).toBe(handlers!.length);
},
);
it("the cookie check refuses every transport but the cookie", () => {
const support = stripComments(
readFileSync(
join(process.cwd(), "src/lib/documents/sources/route-support.ts"),
"utf8",
),
);
expect(support).toContain('auth.authMethod !== "cookie"');
expect(support).toContain('"documents.sources.browserOnly"');
});
});
@@ -161,6 +161,13 @@ const PINNED_AFFORDANCES: Record<
"src/components/documents/document-card.tsx": { "text-success": 1 },
"src/components/documents/document-detail-sheet.tsx": { "toast.success": 2 },
"src/components/documents/documents-view.tsx": { "toast.success": 3 },
// The document picker's per-row result: tinted only for the `imported` /
// `duplicate` outcome the import route returned for that one document, i.e.
// a document the vault now holds. The run's headline goes through
// WrittenOutcomeLine; a row sits inside a button, where its <p> cannot.
"src/components/documents/sources/document-source-picker.tsx": {
"text-success": 1,
},
"src/components/error-details.tsx": { CheckCircle2: 2, "text-success": 1 },
"src/components/illness/episode-menu.tsx": {
CheckCircle2: 2,
@@ -67,6 +67,11 @@ export const AI_ROUTES: Readonly<Record<string, AiRouteEntry>> = {
capabilities: ["documentAi"],
why: "The upload is data and always accepted; only the AI work queued after it follows the capability.",
},
"src/app/api/documents/sources/[system]/import/route.ts": {
kind: "mixed",
capabilities: ["documentAi"],
why: "A picked document is stored whatever AI says; only the summary queued after a fresh insert follows the capability.",
},
"src/app/api/auth/me/documents-auto-ai-read/route.ts": {
kind: "mixed",
capabilities: ["documentAi"],
@@ -51,10 +51,10 @@ vi.mock("@/lib/db", () => {
findFirst: vi.fn(),
},
documentImportKey: {
findUnique: vi.fn(),
findFirst: vi.fn(),
},
documentSourceAlias: {
findUnique: vi.fn(),
findFirst: vi.fn(),
createMany: vi.fn(),
count: vi.fn(),
},
@@ -239,10 +239,10 @@ beforeEach(() => {
[] as never,
);
vi.mocked(prisma.inboundDocument.findFirst).mockResolvedValue(null as never);
vi.mocked(prisma.documentImportKey.findUnique).mockResolvedValue(
vi.mocked(prisma.documentImportKey.findFirst).mockResolvedValue(
null as never,
);
vi.mocked(prisma.documentSourceAlias.findUnique).mockResolvedValue(
vi.mocked(prisma.documentSourceAlias.findFirst).mockResolvedValue(
null as never,
);
vi.mocked(prisma.documentSourceAlias.createMany).mockResolvedValue({
@@ -783,7 +783,7 @@ describe("POST /api/documents/inbound — document import", () => {
});
it("remembers a purged import through the ledger", async () => {
vi.mocked(prisma.documentImportKey.findUnique).mockResolvedValue({
vi.mocked(prisma.documentImportKey.findFirst).mockResolvedValue({
id: "key-1",
} as never);
const res = await post(
@@ -909,6 +909,8 @@ describe("POST /api/documents/inbound — document import", () => {
userId: "user-1",
documentId: "doc-manual",
sourceSystem: "PAPRA",
// Sent without an instance: the alias matches any Papra.
sourceInstance: null,
sourceId: "doc_9",
},
],
@@ -936,7 +938,7 @@ describe("POST /api/documents/inbound — document import", () => {
});
it("resolves a key held as an alias of a deleted document to deleted", async () => {
vi.mocked(prisma.documentSourceAlias.findUnique).mockResolvedValue({
vi.mocked(prisma.documentSourceAlias.findFirst).mockResolvedValue({
document: docRow({ id: "doc-gone", deletedAt: new Date() }),
} as never);
const res = await post(
+80 -280
View File
@@ -32,7 +32,6 @@
* instruction inside it. Storing it does nothing with its contents.
*/
import { Buffer } from "node:buffer";
import { createHash } from "node:crypto";
import { NextResponse } from "next/server";
@@ -51,42 +50,38 @@ import {
sanitiseZodIssues,
} from "@/lib/api-response";
import { getAiCapability } from "@/lib/ai/capabilities/gate";
import { auditLog } from "@/lib/auth/audit";
import { prisma } from "@/lib/db";
import { hashQueryTokens } from "@/lib/documents/content-index";
import {
ingestDocument,
isoDateToUtc,
personalUploadBucket,
UPLOAD_LIMIT_PER_HOUR,
UPLOAD_WINDOW_MS,
} from "@/lib/documents/ingest";
import {
loadConditionLinks,
loadDocumentEncounterLinks,
narrowOwnedEncounterIds,
narrowOwnedEpisodeIds,
} from "@/lib/documents/links";
import {
encryptDocumentContent,
serialiseDocument,
type SerialisableDocument,
} from "@/lib/documents/store";
import { enqueueDocumentIndex } from "@/lib/jobs/document-index";
import { enqueueDocumentSummary } from "@/lib/jobs/document-summary";
import { enqueueDocumentThumbnail } from "@/lib/jobs/document-thumbnail";
import { DOCUMENTS_WRITE_SCOPE } from "@/lib/documents/scopes";
import {
checkSourceLookupRateLimit,
findSourceKey,
rememberSourceAlias,
type SourceKeyMatch,
} from "@/lib/documents/source-key";
import {
acquireDocumentUploadSlot,
detectDocumentType,
resolveDocumentLimits,
resolveDocumentUploadLimitPerHour,
} from "@/lib/documents/upload-policy";
import { withIdempotency } from "@/lib/idempotency";
import { BodyTooLargeError, readBoundedBody } from "@/lib/labs/ocr-upload";
import { linkTargets } from "@/lib/links";
import { annotate } from "@/lib/logging/context";
import { requireModuleEnabled } from "@/lib/modules/gate";
import { isP2002 } from "@/lib/prisma-errors";
import { checkRateLimit, rateLimitHeaders } from "@/lib/rate-limit";
import {
documentCreateSchema,
@@ -99,13 +94,6 @@ import type { Prisma } from "@/generated/prisma/client";
export const dynamic = "force-dynamic";
/**
* A store-only upload touches no provider — the only abuse vector is disk, so
* a generous per-user ceiling is enough (extraction has its own tight gate).
*/
const UPLOAD_LIMIT_PER_HOUR = 60;
const UPLOAD_WINDOW_MS = 60 * 60 * 1000;
/**
* Multipart envelope allowance on top of the per-file cap: boundaries plus
* the small metadata fields (title / kind / documentDate / episodeIds). The
@@ -114,11 +102,6 @@ const UPLOAD_WINDOW_MS = 60 * 60 * 1000;
const MULTIPART_OVERHEAD_BYTES = 64 * 1024;
const UPLOAD_BODY_TIMEOUT_MS = 30_000;
/** Parse a YYYY-MM-DD form field into a UTC midnight Date. */
function isoDateToUtc(value: string): Date {
return new Date(`${value}T00:00:00.000Z`);
}
/** §3.2 — 413 fileTooLarge with the configured limit in `meta`. */
function fileTooLarge(maxFileBytes: number): NextResponse {
return apiError("File is too large.", 413, {
@@ -128,14 +111,6 @@ function fileTooLarge(maxFileBytes: number): NextResponse {
});
}
/** Internal signal: the quota gate inside the insert transaction tripped. */
class QuotaExceededError extends Error {
constructor(public readonly usedBytes: number) {
super("Document quota exceeded");
this.name = "QuotaExceededError";
}
}
/** What a single upload asked for, carried into the annotations. */
interface UploadContext {
userId: string;
@@ -299,14 +274,17 @@ async function processUpload(
const url = new URL(request.url);
const querySystem = url.searchParams.get("sourceSystem");
const queryId = url.searchParams.get("sourceId");
const queryInstance = url.searchParams.get("sourceInstance");
let queryKey: {
sourceSystem: DocumentSourceSystemValue;
sourceId: string;
sourceInstance?: string;
} | null = null;
if (querySystem !== null || queryId !== null) {
const parsedKey = documentSourceKeySchema.safeParse({
sourceSystem: querySystem ?? undefined,
sourceId: queryId ?? undefined,
sourceInstance: queryInstance ?? undefined,
});
if (!parsedKey.success) {
return apiValidationError(
@@ -337,6 +315,7 @@ async function processUpload(
user.id,
queryKey.sourceSystem,
queryKey.sourceId,
queryKey.sourceInstance ?? null,
);
if (match) {
return answerSourceKey(
@@ -357,7 +336,7 @@ async function processUpload(
// draw on. The cookie / wildcard bucket is unchanged.
const bucketKey = scoped
? `documents-upload:token:${auth.session.id}`
: `documents-upload:${user.id}`;
: personalUploadBucket(user.id);
const rl = await checkRateLimit(
bucketKey,
scoped ? resolveDocumentUploadLimitPerHour() : UPLOAD_LIMIT_PER_HOUR,
@@ -433,6 +412,7 @@ async function processUpload(
encounterIds: rawEncounterIds.length > 0 ? rawEncounterIds : undefined,
sourceSystem: formData.get("sourceSystem") ?? undefined,
sourceId: formData.get("sourceId") ?? undefined,
sourceInstance: formData.get("sourceInstance") ?? undefined,
aiRead: formData.get("aiRead") ?? undefined,
});
if (!parsed.success) {
@@ -450,10 +430,14 @@ async function processUpload(
// both they have to agree.
const formSystem = parsed.data.sourceSystem ?? null;
const formId = formSystem ? (parsed.data.sourceId ?? null) : null;
const formInstance = formId ? (parsed.data.sourceInstance ?? null) : null;
if (
queryKey &&
formSystem !== null &&
(formSystem !== queryKey.sourceSystem || formId !== queryKey.sourceId)
(formSystem !== queryKey.sourceSystem ||
formId !== queryKey.sourceId ||
(formInstance !== null &&
formInstance !== (queryKey.sourceInstance ?? null)))
) {
return apiError(
"The source key in the address and in the form differ.",
@@ -463,6 +447,9 @@ async function processUpload(
}
const sourceSystem = queryKey?.sourceSystem ?? formSystem;
const sourceId = queryKey?.sourceId ?? formId;
const sourceInstance = sourceId
? (queryKey?.sourceInstance ?? formInstance ?? null)
: null;
const ctx: UploadContext = {
userId: user.id,
scoped,
@@ -476,7 +463,12 @@ async function processUpload(
// count (the unique index has no `deleted_at` predicate), and past the purge
// the ledger remembers.
if (sourceSystem && sourceId && !queryKey) {
const match = await findSourceKey(user.id, sourceSystem, sourceId);
const match = await findSourceKey(
user.id,
sourceSystem,
sourceId,
sourceInstance,
);
if (match) return answerSourceKey(ctx, match);
}
@@ -486,157 +478,59 @@ async function processUpload(
} catch {
return apiError("Failed to read uploaded file", 400);
}
if (buffer.byteLength > limits.maxFileBytes) {
return fileTooLarge(limits.maxFileBytes);
}
if (buffer.byteLength === 0) {
return apiError("Uploaded file is empty", 422, {
errorCode: "documents.inbound.invalidMetadata",
});
}
// §3.1 — magic-byte classification; the wire Content-Type is never trusted.
const detected = detectDocumentType(
buffer,
typeof file.name === "string" ? file.name : null,
);
if (!detected) {
return apiError("This file type is not supported.", 415, {
errorCode: "documents.inbound.fileType",
reason: "unsupportedType",
});
}
// Pre-linking: every episode id must be a LIVE episode of the caller.
const episodeIds = await narrowOwnedEpisodeIds(
user.id,
parsed.data.episodeIds ?? [],
);
if (episodeIds === null) {
return apiError("Episode not found", 404, {
errorCode: "documents.inbound.episodeNotFound",
});
}
// Same for the visit ids the review step offered. A refusal rather than a
// silent drop: the person saw a visit named and would otherwise watch the
// upload succeed without the link they asked for.
const encounterIds = await narrowOwnedEncounterIds(
user.id,
parsed.data.encounterIds ?? [],
);
if (encounterIds === null) {
return apiError("Visit not found", 404, {
errorCode: "documents.inbound.encounterNotFound",
});
}
// sha256 of the PLAINTEXT for same-user duplicate detection.
const contentSha256 = createHash("sha256").update(buffer).digest("hex");
// Fast-path dedupe check; the partial unique index closes the race below.
const existing = await prisma.inboundDocument.findFirst({
where: { userId: user.id, contentSha256, deletedAt: null },
omit: { contentEncrypted: true },
// Everything from here to the stored row is the shared ingest path
// (`src/lib/documents/ingest.ts`), the one the document picker's import
// stores through as well.
const result = await ingestDocument({
userId: user.id,
scoped,
ipAddress: getClientIp(request),
bytes: buffer,
filename: typeof file.name === "string" ? file.name : null,
title: parsed.data.title ?? null,
kind: parsed.data.kind ?? null,
documentDate: parsed.data.documentDate ?? null,
episodeIds: parsed.data.episodeIds ?? [],
encounterIds: parsed.data.encounterIds ?? [],
sourceSystem,
sourceId,
sourceInstance,
aiDeferred: ctx.aiDeferred,
sourceKeyChecked: true,
limits,
// Only the AI work queued after a fresh insert follows the capability;
// the upload itself is data and always accepted.
documentAi: () => getAiCapability("documentAi"),
});
if (existing) {
// An import sending bytes that are already stored under another key (or
// none) is answered with that document — and the key is remembered for
// it, so once the person deletes the document this key stays deleted too.
if (sourceSystem && sourceId) {
const remembered = await rememberSourceAlias(
user.id,
existing.id,
sourceSystem,
sourceId,
);
if (remembered === "limit") return aliasLimitResponse();
}
return duplicateResponse(ctx, existing);
}
// Quota gate + insert + pre-links in ONE transaction. Usage counts every
// non-purged row — tombstones still hold TOAST bytes, so "deleted" bytes
// are never invisible weight (undo-delete never changes usage).
let document: SerialisableDocument;
try {
document = await prisma.$transaction(async (tx) => {
// Serialise the quota gate per user: without this, N concurrent
// uploads all read the same SUM before any of them commits and the
// quota can be overshot by up to N × cap in one burst. The advisory
// lock is transaction-scoped (released on commit/rollback) and keyed
// on the user id, so uploads by different users never queue on each
// other.
// (`pg_advisory_xact_lock` returns void, which the client cannot
// deserialize as a column — selecting FROM it yields a plain int row.)
await tx.$queryRaw`
SELECT 1 AS locked
FROM pg_advisory_xact_lock(hashtextextended('documents-quota:' || ${user.id}, 0))
`;
const rows = await tx.$queryRaw<Array<{ used: bigint }>>`
SELECT COALESCE(SUM(byte_size), 0)::bigint AS used
FROM inbound_documents
WHERE user_id = ${user.id}
`;
const usedBytes = Number(rows[0]?.used ?? 0);
if (usedBytes + buffer.byteLength > limits.quotaBytes) {
throw new QuotaExceededError(usedBytes);
}
const { content, codec } = encryptDocumentContent(buffer);
// No mass assignment — every column is set field-by-field; `userId`
// comes from the session, never the body. `documentDate` defaults to
// the upload day so display == sort == filter (user-editable later).
const created = await tx.inboundDocument.create({
data: {
userId: user.id,
kind: parsed.data.kind ?? "OTHER",
// Stored plaintext on purpose (mirrors `filename`) so the list can
// ILIKE-search + ORDER BY it. It MAY hold PHI the user types; that
// is the accepted tradeoff for server-side search/sort — the
// document body stays encrypted.
title: parsed.data.title ?? null,
filename:
typeof file.name === "string" ? file.name.slice(0, 255) : null,
mimeType: detected.mimeType,
byteSize: buffer.byteLength,
contentEncrypted: content,
contentCodec: codec,
contentSha256,
status: "STORED",
documentDate: parsed.data.documentDate
? isoDateToUtc(parsed.data.documentDate)
: new Date(),
sourceSystem,
sourceId,
aiReadDeferred: ctx.aiDeferred,
},
omit: { contentEncrypted: true },
switch (result.kind) {
case "sourceKey":
return answerSourceKey(ctx, result.match);
case "tooLarge":
return fileTooLarge(result.maxFileBytes);
case "empty":
return apiError("Uploaded file is empty", 422, {
errorCode: "documents.inbound.invalidMetadata",
});
if (episodeIds.length > 0) {
await linkTargets(tx, {
userId: user.id,
sourceKind: "document",
sourceId: created.id,
targetKind: "conditionEpisode",
targetIds: episodeIds,
});
}
if (encounterIds.length > 0) {
await linkTargets(tx, {
userId: user.id,
sourceKind: "document",
sourceId: created.id,
targetKind: "encounter",
targetIds: encounterIds,
});
}
return created;
});
} catch (err) {
if (err instanceof QuotaExceededError) {
case "unsupportedType":
return apiError("This file type is not supported.", 415, {
errorCode: "documents.inbound.fileType",
reason: "unsupportedType",
});
case "episodeNotFound":
return apiError("Episode not found", 404, {
errorCode: "documents.inbound.episodeNotFound",
});
case "encounterNotFound":
return apiError("Visit not found", 404, {
errorCode: "documents.inbound.encounterNotFound",
});
case "aliasLimit":
return aliasLimitResponse();
case "duplicate":
return duplicateResponse(ctx, result.document);
case "quotaExceeded":
// The figures are the owner's; a write-only token learns only that the
// vault is full.
return apiError("Storage quota exceeded.", 413, {
@@ -644,107 +538,13 @@ async function processUpload(
reason: "quotaExceeded",
...(scoped
? {}
: { quotaBytes: limits.quotaBytes, usedBytes: err.usedBytes }),
: { quotaBytes: result.quotaBytes, usedBytes: result.usedBytes }),
});
}
if (isP2002(err)) {
// A racing upload won one of the two partial unique indexes — the same
// source key, or the same bytes. Surface the winner exactly as the fast
// paths above would have.
if (sourceSystem && sourceId) {
const match = await findSourceKey(user.id, sourceSystem, sourceId);
if (match) return answerSourceKey(ctx, match);
}
const winner = await prisma.inboundDocument.findFirst({
where: { userId: user.id, contentSha256, deletedAt: null },
omit: { contentEncrypted: true },
});
if (winner) {
if (sourceSystem && sourceId) {
const remembered = await rememberSourceAlias(
user.id,
winner.id,
sourceSystem,
sourceId,
);
if (remembered === "limit") return aliasLimitResponse();
}
return duplicateResponse(ctx, winner);
}
}
throw err;
}
await auditLog("documents.inbound.store", {
userId: user.id,
ipAddress: getClientIp(request),
details: {
documentId: document.id,
mime: detected.mimeType,
...(scoped ? { scoped: true } : {}),
...(sourceSystem ? { sourceSystem } : {}),
},
});
annotate({
action: { name: "documents.vault.upload" },
meta: {
documentId: document.id,
byteSize: document.byteSize,
servingClass: detected.servingClass,
linked: episodeIds.length,
linkedVisits: encounterIds.length,
scoped,
sourceSystem,
aiDeferred: ctx.aiDeferred,
},
});
// Auto-index the freshly stored document for content search: enqueue a
// fire-and-forget background job (provider-first, local text-layer fallback).
// The upload response never blocks on or fails because of indexing — the
// enqueue is not awaited and swallows its own errors (a missing boss or a
// transient send failure is a silent no-op). Only fresh inserts enqueue — a
// duplicate upload returns early above and never reaches here.
//
// `aiRead=defer` narrows it to the local text layer: an import of a whole
// archive must not turn into one provider call per file. The person reads
// them later, deliberately, from the document itself.
if (ctx.aiDeferred) {
void enqueueDocumentIndex(user.id, document.id, { localOnly: true });
} else {
void enqueueDocumentIndex(user.id, document.id);
}
// Render a preview thumbnail in the background too (pure local compute — no
// egress). Same fire-and-forget contract: the upload never blocks on or fails
// because of it, and a dropped enqueue is recoverable via the boot backfill.
// Only fresh inserts reach here (a duplicate returns early above).
void enqueueDocumentThumbnail(user.id, document.id);
// Summarise the freshly stored document in the background — but ONLY when the
// `documentsAutoAiRead` opt-in is ON (the job re-checks it and the egress
// consent, and no-ops otherwise) and the `documentAi` capability is open for
// this record. The upload itself never depends on AI: the document is stored
// above whatever the capability says, and the index job still runs its
// provider-free text-layer path. The persisted summary shows on the detail
// view. Same fire-and-forget contract: the upload never blocks on or fails
// because of it. Only fresh inserts reach here (a duplicate returns early).
//
// A deferred upload skips it outright and keeps `summaryState: NONE`, so
// the detail sheet offers "Generate summary" rather than a pending state.
const documentAi = ctx.aiDeferred
? { available: false as const, reason: "deferred" }
: await getAiCapability("documentAi");
if (documentAi.available) {
void enqueueDocumentSummary(user.id, document.id);
} else {
annotate({
action: { name: "documents.summary.enqueueSkipped" },
meta: { documentId: document.id, reason: documentAi.reason },
});
case "stored":
break;
}
const document = result.document;
if (scoped) return receiptResponse(document.id, 201, { duplicate: false });
const [links, visitLinks] = await Promise.all([
@@ -57,6 +57,7 @@ export const GET = apiHandler(async (request: Request) => {
const parsed = documentSourceKeySchema.safeParse({
sourceSystem: url.searchParams.get("sourceSystem") ?? undefined,
sourceId: url.searchParams.get("sourceId") ?? undefined,
sourceInstance: url.searchParams.get("sourceInstance") ?? undefined,
});
if (!parsed.success) {
return apiValidationError(
@@ -71,6 +72,7 @@ export const GET = apiHandler(async (request: Request) => {
user.id,
parsed.data.sourceSystem,
parsed.data.sourceId,
parsed.data.sourceInstance ?? null,
);
annotate({
action: { name: "documents.vault.sourceLookup" },
@@ -0,0 +1,385 @@
/**
* Document picker (#1038): import one picked document from a connected
* archive into the vault.
*
* One document per request, so the sheet can show each one's progress and
* result, and no single request holds more than one file in memory. The path:
*
* 1. the link target, when the picker was opened from a record, must be one
* of the person's own live records (404 before anything leaves the
* server);
* 2. the source key answers first, without a download: a document already
* imported (or already holding this key as an alias) is a `duplicate`,
* and one the person deleted here is `deleted` — nothing is fetched and
* nothing is stored, exactly as the token upload answers the same key;
* 3. only then is the source asked (on the search allowance, like every
* request to it) and an upload slot taken. One import per key and person
* runs at a time under an advisory lock, so a double click downloads the
* document once. The details and the original file are fetched (bounded
* by the vault's per-file cap); only a file that arrived is charged to
* the vault's own upload allowance (60 an hour, shared with uploads from
* the web and the phone), and it goes through `ingestDocument`, the same
* function the upload route stores through: type check, content dedup,
* quota, provenance (`sourceSystem` / `sourceId` / `sourceInstance`), the
* background jobs;
* 4. the document is linked to the record through `src/lib/links/`.
*
* AI reading follows the person's own setting, unlike the token upload's
* default of holding it back: the person picked this document by hand, as
* deliberately as an upload. The hourly allowance and the daily AI budget
* bound it either way.
*
* Cookie-only and owner-only; see `admitDocumentSourceCaller`.
*/
import { Buffer } from "node:buffer";
import { NextRequest, NextResponse } from "next/server";
import { apiHandler, requireAuth } from "@/lib/api-handler";
import {
apiError,
apiValidationError,
getClientIp,
safeJson,
sanitiseZodIssues,
} from "@/lib/api-response";
import { getAiCapability } from "@/lib/ai/capabilities/gate";
import { prisma } from "@/lib/db";
import {
ingestDocument,
personalUploadBucket,
UPLOAD_LIMIT_PER_HOUR,
UPLOAD_WINDOW_MS,
} from "@/lib/documents/ingest";
import {
checkSourceLookupRateLimit,
findSourceKey,
type SourceKeyMatch,
} from "@/lib/documents/source-key";
import type { SourceClient } from "@/lib/documents/sources/clients";
import { clientFor, loadConnection } from "@/lib/documents/sources/connections";
import {
admitDocumentSourceCaller,
checkSearchRateLimit,
responseForSourceFailure,
sourceErrorResponse,
systemParam,
} from "@/lib/documents/sources/route-support";
import {
isSourceDocumentId,
type DocumentImportOutcome,
type DocumentPickerLinkKind,
type DocumentPickerSystem,
type DocumentSourceImportDto,
} from "@/lib/documents/sources/types";
import {
acquireDocumentUploadSlot,
resolveDocumentLimits,
} from "@/lib/documents/upload-policy";
import { linkTargets } from "@/lib/links";
import { annotate } from "@/lib/logging/context";
import { checkRateLimit, rateLimitHeaders } from "@/lib/rate-limit";
import { documentSourceImportSchema } from "@/lib/validations/document-sources";
import type { InboundDocumentKindValue } from "@/lib/validations/inbound-documents";
export const dynamic = "force-dynamic";
type RouteParams = { params: Promise<{ system: string }> };
/** How long an import waits for a database connection to hold its lock on. */
const IMPORT_LOCK_WAIT_MS = 10_000;
/** Longest a single import may hold its key: the download plus the store. */
const IMPORT_LOCK_HOLD_MS = 150_000;
interface LinkTarget {
kind: DocumentPickerLinkKind;
id: string;
}
/** Is the record the picker was opened from one of the person's live ones? */
async function ownsLinkTarget(
userId: string,
link: LinkTarget,
): Promise<boolean> {
const where = { id: link.id, userId, deletedAt: null };
const select = { id: true } as const;
switch (link.kind) {
case "conditionEpisode":
return (
(await prisma.illnessEpisode.findFirst({ where, select })) !== null
);
case "encounter":
return (await prisma.encounter.findFirst({ where, select })) !== null;
case "vaccination":
return (
(await prisma.vaccinationRecord.findFirst({ where, select })) !== null
);
}
}
/** File the document against the record, from the record's side. */
async function linkDocument(
userId: string,
link: LinkTarget | undefined,
documentId: string,
): Promise<boolean> {
if (!link) return false;
const result = await prisma.$transaction((tx) =>
linkTargets(tx, {
userId,
sourceKind: link.kind,
sourceId: link.id,
targetKind: "document",
targetIds: [documentId],
}),
);
return !result.unknownSource && result.unknownTargetIds.length === 0;
}
function respond(
system: string,
outcome: DocumentImportOutcome,
documentId: string | null,
linked: boolean,
): NextResponse {
annotate({
action: { name: "documents.sources.import" },
meta: { system, outcome, linked, documentId },
});
const data: DocumentSourceImportDto = { outcome, documentId, linked };
return NextResponse.json(
{ data, error: null },
{ status: outcome === "imported" ? 201 : 200 },
);
}
/** A key the vault already holds: link a live one, never revive a deleted one. */
async function answerHeldKey(
system: string,
userId: string,
link: LinkTarget | undefined,
match: SourceKeyMatch,
): Promise<NextResponse> {
if (match.state === "deleted") {
return respond(system, "deleted", match.id, false);
}
const linked = await linkDocument(userId, link, match.document.id);
return respond(system, "duplicate", match.document.id, linked);
}
export const POST = apiHandler(
async (request: NextRequest, { params }: RouteParams) => {
const auth = await requireAuth();
const refused = await admitDocumentSourceCaller(auth);
if (refused) return refused;
const system = await systemParam(params);
if (system instanceof Response) return system;
const userId = auth.user.id;
const { data: body, error: jsonError } = await safeJson(request, {
maxBytes: 4 * 1024,
});
if (jsonError) return jsonError;
const parsed = documentSourceImportSchema.safeParse(body);
if (!parsed.success) {
return apiValidationError(
"Invalid import request",
sanitiseZodIssues(parsed.error.issues),
422,
);
}
const { sourceId, kind, link } = parsed.data;
// Only an id this system could have issued goes into a request path.
if (!isSourceDocumentId(system, sourceId)) {
return apiError("Not a document id of this archive.", 422);
}
const row = await loadConnection(userId, system);
if (!row) return sourceErrorResponse("notConnected");
let client;
try {
client = clientFor(row);
} catch (err) {
return responseForSourceFailure(err);
}
const instance = client.instance;
if (link && !(await ownsLinkTarget(userId, link))) {
return sourceErrorResponse("linkTargetNotFound");
}
// The source key answers before anything is downloaded or charged, on the
// same metered lookup allowance the upload's key check draws on.
const lookupRl = await checkSourceLookupRateLimit(
false,
auth.session.id,
userId,
);
if (!lookupRl.allowed) {
return sourceErrorResponse(
"rateLimited",
undefined,
rateLimitHeaders(lookupRl),
);
}
const held = await findSourceKey(userId, system, sourceId, instance);
if (held) return answerHeldKey(system, userId, link, held);
// A request to the source counts against the search allowance like any
// other; the upload allowance is charged only once a file arrived.
const fetchRl = await checkSearchRateLimit(userId);
if (!fetchRl.allowed) {
return sourceErrorResponse(
"rateLimited",
undefined,
rateLimitHeaders(fetchRl),
);
}
const release = acquireDocumentUploadSlot(userId);
if (!release) {
return apiError(
"Too many uploads are already in progress. Try again shortly.",
429,
{
errorCode: "documents.inbound.uploadBusy",
reason: "uploadBusy",
headers: { "Retry-After": "1" },
},
);
}
try {
// One import of one key at a time per person: a second tab or a double
// click waits here, then finds the key the first one stored, so the
// document is downloaded once. The lock is transaction-scoped and the
// transaction holds nothing else; the store runs in its own.
return await prisma.$transaction(
async (tx) => {
await tx.$queryRaw`
SELECT 1 AS locked
FROM pg_advisory_xact_lock(hashtextextended(${`documents-import:${userId}:${system}:${instance}:${sourceId}`}, 0))
`;
const raced = await findSourceKey(userId, system, sourceId, instance);
if (raced) return answerHeldKey(system, userId, link, raced);
return importOne({
request,
client,
userId,
system,
instance,
sourceId,
kind: kind ?? null,
link,
});
},
{ maxWait: IMPORT_LOCK_WAIT_MS, timeout: IMPORT_LOCK_HOLD_MS },
);
} finally {
release();
}
},
);
/** Fetch one document and store it; the caller holds the key's lock. */
async function importOne(args: {
request: Request;
client: SourceClient;
userId: string;
system: DocumentPickerSystem;
instance: string;
sourceId: string;
kind: InboundDocumentKindValue | null;
link: LinkTarget | undefined;
}): Promise<Response> {
const { request, client, userId, system, instance, sourceId, kind, link } =
args;
const limits = await resolveDocumentLimits(userId);
let meta;
let file;
try {
meta = await client.document(sourceId);
file = await client.download(sourceId, limits.maxFileBytes);
} catch (err) {
return responseForSourceFailure(err);
}
const rl = await checkRateLimit(
personalUploadBucket(userId),
UPLOAD_LIMIT_PER_HOUR,
UPLOAD_WINDOW_MS,
);
if (!rl.allowed) {
return apiError("Too many uploads. Try again later.", 429, {
errorCode: "documents.inbound.rateLimited",
headers: rateLimitHeaders(rl),
});
}
const result = await ingestDocument({
userId,
scoped: false,
ipAddress: getClientIp(request),
bytes: Buffer.from(file.bytes),
filename: meta.filename ?? file.filename,
title: meta.title,
kind,
documentDate: meta.date,
episodeIds: [],
encounterIds: [],
sourceSystem: system,
sourceId,
sourceInstance: instance,
aiDeferred: false,
sourceKeyChecked: true,
limits,
// The import is data; only the summary queued after a fresh insert
// follows the capability (and the person's auto-read setting).
documentAi: () => getAiCapability("documentAi"),
});
switch (result.kind) {
case "sourceKey":
return answerHeldKey(system, userId, link, result.match);
case "duplicate": {
// The same bytes were already in the vault (an upload, or another
// archive); the key is now remembered for that document.
const linked = await linkDocument(userId, link, result.document.id);
return respond(system, "duplicate", result.document.id, linked);
}
case "stored": {
const linked = await linkDocument(userId, link, result.document.id);
return respond(system, "imported", result.document.id, linked);
}
case "tooLarge":
return apiError("File is too large.", 413, {
errorCode: "documents.inbound.fileTooLarge",
reason: "fileTooLarge",
maxFileBytes: result.maxFileBytes,
});
case "empty":
case "unsupportedType":
return apiError("This file type is not supported.", 415, {
errorCode: "documents.inbound.fileType",
reason: "unsupportedType",
});
case "quotaExceeded":
return apiError("Storage quota exceeded.", 413, {
errorCode: "documents.inbound.quotaExceeded",
reason: "quotaExceeded",
quotaBytes: result.quotaBytes,
usedBytes: result.usedBytes,
});
case "aliasLimit":
return apiError(
"This file is already stored under too many source ids.",
409,
{ errorCode: "documents.inbound.sourceAliasLimit" },
);
case "episodeNotFound":
case "encounterNotFound":
// No pre-links are passed on this path; unreachable, but a result
// the switch does not name must not fall through silently.
return sourceErrorResponse("linkTargetNotFound");
}
}
@@ -0,0 +1,204 @@
/**
* Document picker (#1038): save or remove the person's connection to one
* document archive (`paperless` or `papra`).
*
* PUT validates the base address against the operator's list
* (`DOCUMENT_SOURCE_ORIGINS`), tests the connection live before storing
* anything (a wrong token or an unreachable host is reported here, not on the
* first search), then stores the token encrypted. A PUT without a token keeps
* the stored one, so the base path or the Papra organization can change
* without typing the key again, but only on the same origin: a new origin, and
* the first save, must carry it (422 `documents.sources.tokenRequired`). The response is the
* connection without its token, like every read of it.
*
* DELETE removes the row, and with it the token.
*
* Cookie-only and owner-only; see `admitDocumentSourceCaller`.
*/
import { NextRequest } from "next/server";
import { apiHandler, requireAuth } from "@/lib/api-handler";
import {
apiError,
apiSuccess,
apiValidationError,
getClientIp,
safeJson,
sanitiseZodIssues,
} from "@/lib/api-response";
import { auditLog } from "@/lib/auth/audit";
import { decrypt, encrypt } from "@/lib/crypto";
import { prisma } from "@/lib/db";
import { sourceClient } from "@/lib/documents/sources/clients";
import {
loadConnection,
toConnectionDto,
} from "@/lib/documents/sources/connections";
import { DocumentSourceError } from "@/lib/documents/sources/http";
import { evaluateSourceBaseUrl } from "@/lib/documents/sources/origins";
import {
admitDocumentSourceCaller,
checkConnectRateLimit,
responseForSourceFailure,
sourceErrorResponse,
systemParam,
} from "@/lib/documents/sources/route-support";
import { annotate } from "@/lib/logging/context";
import { rateLimitHeaders } from "@/lib/rate-limit";
import { documentSourceSaveSchema } from "@/lib/validations/document-sources";
export const dynamic = "force-dynamic";
type RouteParams = { params: Promise<{ system: string }> };
export const PUT = apiHandler(
async (request: NextRequest, { params }: RouteParams) => {
const auth = await requireAuth();
const refused = await admitDocumentSourceCaller(auth);
if (refused) return refused;
const system = await systemParam(params);
if (system instanceof Response) return system;
const userId = auth.user.id;
annotate({ action: { name: "documents.sources.save" }, meta: { system } });
const rl = await checkConnectRateLimit(userId);
if (!rl.allowed) {
return sourceErrorResponse(
"rateLimited",
undefined,
rateLimitHeaders(rl),
);
}
const { data: body, error: jsonError } = await safeJson(request, {
maxBytes: 8 * 1024,
});
if (jsonError) return jsonError;
const parsed = documentSourceSaveSchema.safeParse(body);
if (!parsed.success) {
return apiValidationError(
"Invalid connection details",
sanitiseZodIssues(parsed.error.issues),
422,
);
}
const verdict = evaluateSourceBaseUrl(parsed.data.baseUrl);
if (!verdict.ok) {
return verdict.reason === "notAllowed"
? sourceErrorResponse("originNotAllowed")
: apiError("Enter the full address, starting with https://.", 422, {
errorCode: "documents.sources.invalidAddress",
});
}
const organizationId =
system === "PAPRA" ? (parsed.data.organizationId ?? null) : null;
if (system === "PAPRA" && !organizationId) {
return apiError("Papra needs the organization id.", 422, {
errorCode: "documents.sources.organizationRequired",
});
}
const existing = await loadConnection(userId, system);
// The stored token is only ever sent back to the origin it was saved
// for. A new address on another origin needs the token typed again, or
// saving an address would hand the old archive's token to whatever the
// new one is.
const reusable =
existing !== null &&
evaluateSourceBaseUrl(existing.baseUrl, new Set([verdict.origin])).ok;
const token =
parsed.data.token ??
(existing && reusable ? decrypt(existing.tokenEncrypted) : null);
if (!token) {
return apiError("Enter the API token.", 422, {
errorCode: "documents.sources.tokenRequired",
});
}
// Live check before anything is stored: the row only ever holds a
// connection that answered once.
try {
await sourceClient({
system,
origin: verdict.origin,
baseUrl: verdict.baseUrl,
organizationId,
token,
}).test();
} catch (err) {
if (err instanceof DocumentSourceError) {
annotate({ meta: { refused: err.code } });
}
return responseForSourceFailure(err);
}
const now = new Date();
const tokenEncrypted = encrypt(token);
// Field by field; `userId` from the session, never the body.
const saved = await prisma.documentSourceConnection.upsert({
where: { userId_system: { userId, system } },
create: {
userId,
system,
baseUrl: verdict.baseUrl,
organizationId,
tokenEncrypted,
lastVerifiedAt: now,
},
update: {
baseUrl: verdict.baseUrl,
organizationId,
tokenEncrypted,
lastVerifiedAt: now,
},
select: {
system: true,
baseUrl: true,
organizationId: true,
lastVerifiedAt: true,
},
});
await auditLog("documents.sources.connect", {
userId,
ipAddress: getClientIp(request),
details: { system, origin: verdict.origin, replaced: existing !== null },
});
return apiSuccess(toConnectionDto(saved));
},
);
export const DELETE = apiHandler(
async (request: NextRequest, { params }: RouteParams) => {
const auth = await requireAuth();
// Neither the list nor the module is needed to forget a connection: an
// operator who removed the origin, or a person who switched Documents
// off, must still be able to delete the stored token.
const refused = await admitDocumentSourceCaller(auth, {
needsList: false,
needsModule: false,
});
if (refused) return refused;
const system = await systemParam(params);
if (system instanceof Response) return system;
annotate({
action: { name: "documents.sources.disconnect" },
meta: { system },
});
const { count } = await prisma.documentSourceConnection.deleteMany({
where: { userId: auth.user.id, system },
});
if (count > 0) {
await auditLog("documents.sources.disconnect", {
userId: auth.user.id,
ipAddress: getClientIp(request),
details: { system },
});
}
return apiSuccess({ disconnected: count > 0 });
},
);
@@ -0,0 +1,159 @@
/**
* Document picker (#1038): search one connected document archive.
*
* `q` searches the document's name (the title in Paperless-ngx, the name in
* Papra), `tag` narrows to one of the source's tags by its id, `from` / `to`
* to a date range (inclusive), `page` pages through 25 at a time. The answer
* is a minimal row per document (source id, title, date, tag names, size when
* the source says it) plus what HealthLog already holds under that source key:
* `new`, `imported` (with the vault document's id) or `deleted` (the person
* deleted it here; importing it again is refused). Nothing is stored.
*
* When the source ignores a filter and the local re-check empties a page, the
* route reads on (up to five source pages) and answers with the last page it
* read, so the client's "load more" continues after it.
*
* Sixty searches a minute, shared with the tag read. Cookie-only and
* owner-only; see `admitDocumentSourceCaller`.
*/
import { NextRequest } from "next/server";
import { apiHandler, requireAuth } from "@/lib/api-handler";
import {
apiSuccess,
apiValidationError,
sanitiseZodIssues,
} from "@/lib/api-response";
import { findSourceKeyStates } from "@/lib/documents/source-key";
import type { SourceListItem } from "@/lib/documents/sources/clients";
import { openConnection } from "@/lib/documents/sources/connections";
import {
admitDocumentSourceCaller,
checkSearchRateLimit,
responseForSourceFailure,
sourceErrorResponse,
systemParam,
} from "@/lib/documents/sources/route-support";
import {
DOCUMENT_PICKER_MAX_PAGE,
type DocumentSourceResultDto,
type DocumentSourceSearchDto,
} from "@/lib/documents/sources/types";
import { annotate } from "@/lib/logging/context";
import { rateLimitHeaders } from "@/lib/rate-limit";
import { documentSourceSearchSchema } from "@/lib/validations/document-sources";
export const dynamic = "force-dynamic";
/** Source pages one search request may read to find a non-empty one. */
const SEARCH_PAGES_PER_REQUEST = 5;
type RouteParams = { params: Promise<{ system: string }> };
export const GET = apiHandler(
async (request: NextRequest, { params }: RouteParams) => {
const auth = await requireAuth();
const refused = await admitDocumentSourceCaller(auth);
if (refused) return refused;
const system = await systemParam(params);
if (system instanceof Response) return system;
const userId = auth.user.id;
const url = new URL(request.url);
const parsed = documentSourceSearchSchema.safeParse({
q: url.searchParams.get("q") ?? undefined,
tag: url.searchParams.get("tag") ?? undefined,
from: url.searchParams.get("from") ?? undefined,
to: url.searchParams.get("to") ?? undefined,
page: url.searchParams.get("page") ?? undefined,
});
if (!parsed.success) {
return apiValidationError(
"Invalid search",
sanitiseZodIssues(parsed.error.issues),
422,
);
}
const { q, tag, from, to, page } = parsed.data;
const rl = await checkSearchRateLimit(userId);
if (!rl.allowed) {
return sourceErrorResponse(
"rateLimited",
undefined,
rateLimitHeaders(rl),
);
}
// A page the local tag/date re-check emptied is not an answer: the next
// source page may hold matches. Read on, a bounded number of pages, and
// report the last one read so "load more" continues after it.
let found: { items: SourceListItem[]; hasMore: boolean } = {
items: [],
hasMore: false,
};
let lastPage = page;
let instance: string;
try {
const client = await openConnection(userId, system);
instance = client.instance;
for (
let next = page;
next < page + SEARCH_PAGES_PER_REQUEST &&
next <= DOCUMENT_PICKER_MAX_PAGE;
next += 1
) {
found = await client.search({
q,
tagId: tag ?? null,
from: from ?? null,
to: to ?? null,
page: next,
});
lastPage = next;
if (found.items.length > 0 || !found.hasMore) break;
}
} catch (err) {
return responseForSourceFailure(err);
}
const states = await findSourceKeyStates(
userId,
system,
found.items.map((item) => item.sourceId),
instance,
);
const results: DocumentSourceResultDto[] = found.items.map((item) => {
const held = states.get(item.sourceId);
return {
sourceId: item.sourceId,
title: item.title,
date: item.date,
tags: item.tags,
sizeBytes: item.sizeBytes,
state: held?.state ?? "new",
documentId: held?.state === "imported" ? held.documentId : null,
};
});
// Counts only: the query and the titles are the person's own words and
// stay out of the wide event.
annotate({
action: { name: "documents.sources.search" },
meta: {
system,
page,
pagesRead: lastPage - page + 1,
count: results.length,
filtered: Boolean(q || tag || from || to),
},
});
const body: DocumentSourceSearchDto = {
results,
page: lastPage,
hasMore: found.hasMore && lastPage < DOCUMENT_PICKER_MAX_PAGE,
};
return apiSuccess(body);
},
);
@@ -0,0 +1,52 @@
/**
* Document picker (#1038): the tags of one connected document archive, for the
* picker's tag filter. The first 250 by name, as `{ id, name }` in the
* source's own ids. Shares the search allowance (60 a minute).
*
* Cookie-only and owner-only; see `admitDocumentSourceCaller`.
*/
import { NextRequest } from "next/server";
import { apiHandler, requireAuth } from "@/lib/api-handler";
import { apiSuccess } from "@/lib/api-response";
import { openConnection } from "@/lib/documents/sources/connections";
import {
admitDocumentSourceCaller,
checkSearchRateLimit,
responseForSourceFailure,
sourceErrorResponse,
systemParam,
} from "@/lib/documents/sources/route-support";
import { annotate } from "@/lib/logging/context";
import { rateLimitHeaders } from "@/lib/rate-limit";
export const dynamic = "force-dynamic";
type RouteParams = { params: Promise<{ system: string }> };
export const GET = apiHandler(
async (_request: NextRequest, { params }: RouteParams) => {
const auth = await requireAuth();
const refused = await admitDocumentSourceCaller(auth);
if (refused) return refused;
const system = await systemParam(params);
if (system instanceof Response) return system;
annotate({ action: { name: "documents.sources.tags" }, meta: { system } });
const rl = await checkSearchRateLimit(auth.user.id);
if (!rl.allowed) {
return sourceErrorResponse(
"rateLimited",
undefined,
rateLimitHeaders(rl),
);
}
try {
const client = await openConnection(auth.user.id, system);
return apiSuccess({ tags: await client.tags() });
} catch (err) {
return responseForSourceFailure(err);
}
},
);
@@ -0,0 +1,71 @@
/**
* Document picker (#1038): test the saved connection to one document archive.
*
* One request to the source with the stored token, answered `{ ok, latencyMs }`
* or with the reason it failed (`meta.errorCode`, and `meta.upstreamStatus`
* when the source answered at all; its body is never passed on). A success
* stamps `lastVerifiedAt`. Shares the save's allowance of ten a minute.
*
* Cookie-only and owner-only; see `admitDocumentSourceCaller`.
*/
import { NextRequest } from "next/server";
import { apiHandler, requireAuth } from "@/lib/api-handler";
import { apiSuccess } from "@/lib/api-response";
import {
markVerified,
openConnection,
} from "@/lib/documents/sources/connections";
import { DocumentSourceError } from "@/lib/documents/sources/http";
import {
admitDocumentSourceCaller,
checkConnectRateLimit,
responseForSourceFailure,
sourceErrorResponse,
systemParam,
} from "@/lib/documents/sources/route-support";
import { annotate } from "@/lib/logging/context";
import { rateLimitHeaders } from "@/lib/rate-limit";
export const dynamic = "force-dynamic";
type RouteParams = { params: Promise<{ system: string }> };
export const POST = apiHandler(
async (_request: NextRequest, { params }: RouteParams) => {
const auth = await requireAuth();
const refused = await admitDocumentSourceCaller(auth);
if (refused) return refused;
const system = await systemParam(params);
if (system instanceof Response) return system;
const userId = auth.user.id;
annotate({ action: { name: "documents.sources.test" }, meta: { system } });
const rl = await checkConnectRateLimit(userId);
if (!rl.allowed) {
return sourceErrorResponse(
"rateLimited",
undefined,
rateLimitHeaders(rl),
);
}
const start = performance.now();
try {
const client = await openConnection(userId, system);
await client.test();
} catch (err) {
if (err instanceof DocumentSourceError) {
annotate({ meta: { refused: err.code } });
}
return responseForSourceFailure(err);
}
const latencyMs = Math.round(performance.now() - start);
const verifiedAt = await markVerified(userId, system);
return apiSuccess({
ok: true,
latencyMs,
lastVerifiedAt: verifiedAt.toISOString(),
});
},
);
+61
View File
@@ -0,0 +1,61 @@
/**
* Document picker (#1038): which document archives this person can pick from.
*
* `available` says whether the operator listed any origin in
* `DOCUMENT_SOURCE_ORIGINS`; `allowedOrigins` lists them so the settings card
* can say which addresses will be accepted; `connections` are the person's own
* saved connections, never with their token (`hasToken` is the literal true).
* No request leaves the server here.
*
* Cookie-only and owner-only like every route under this path
* (`admitDocumentSourceCaller`), but it needs neither the documents module nor
* the list: with the list unset it answers 200 `available: false` (plus any
* rows still stored, so they can be deleted) rather than 404, because this is
* the one read the client makes to decide what to show.
*/
import { apiHandler, requireAuth } from "@/lib/api-handler";
import { apiSuccess } from "@/lib/api-response";
import {
listConnections,
toConnectionDto,
} from "@/lib/documents/sources/connections";
import {
documentSourcesAvailable,
listedDocumentSourceOrigins,
} from "@/lib/documents/sources/origins";
import type { DocumentSourcesStatusDto } from "@/lib/documents/sources/types";
import { admitDocumentSourceCaller } from "@/lib/documents/sources/route-support";
import { annotate } from "@/lib/logging/context";
export const dynamic = "force-dynamic";
export const GET = apiHandler(async () => {
const auth = await requireAuth();
const refused = await admitDocumentSourceCaller(auth, {
needsList: false,
needsModule: false,
});
if (refused) return refused;
annotate({ action: { name: "documents.sources.status" } });
// Rows are listed even with the picker off: a connection whose origin the
// operator removed (or whose list is gone) must stay visible so it can be
// deleted. `originAllowed` is false for each of them then.
const rows = await listConnections(auth.user.id);
if (!documentSourcesAvailable()) {
const off: DocumentSourcesStatusDto = {
available: false,
allowedOrigins: [],
connections: rows.map(toConnectionDto),
};
return apiSuccess(off);
}
const status: DocumentSourcesStatusDto = {
available: true,
allowedOrigins: listedDocumentSourceOrigins(),
connections: rows.map(toConnectionDto),
};
return apiSuccess(status);
});
+12 -7
View File
@@ -67,6 +67,7 @@ import { DocumentShareSheet } from "./document-share-sheet";
import { DocumentFilterBar, type ConditionChip } from "./document-filter-bar";
import { DocumentTimeline } from "./document-timeline";
import { UploadZone } from "./upload-zone";
import { ImportFromSourceButton } from "./sources/import-from-source-button";
import { useReindexAll } from "./use-content-index";
import { useDocumentUpload } from "./use-document-upload";
import { usePageFileDrop } from "./use-page-file-drop";
@@ -876,13 +877,17 @@ export function DocumentsView() {
description={t("documents.subtitle")}
actions={
canManageDocuments ? (
<Button
className="min-h-11 sm:min-h-9"
onClick={() => uploadInputRef.current?.click()}
>
<Upload className="size-4" aria-hidden />
{t("documents.pageUpload")}
</Button>
<div className="flex flex-wrap items-center gap-2">
{/* Renders only with a connected Paperless-ngx or Papra. */}
<ImportFromSourceButton className="min-h-11 sm:min-h-9" />
<Button
className="min-h-11 sm:min-h-9"
onClick={() => uploadInputRef.current?.click()}
>
<Upload className="size-4" aria-hidden />
{t("documents.pageUpload")}
</Button>
</div>
) : null
}
/>
@@ -32,6 +32,7 @@ import { useFormatters, useTranslations } from "@/lib/i18n/context";
import { queryKeys } from "@/lib/query-keys";
import type { InboundDocumentDto } from "@/lib/validations/inbound-documents";
import { DocumentLinkPicker } from "./document-link-picker";
import { ImportFromSourceButton } from "./sources/import-from-source-button";
import { DOCUMENT_KIND_ICONS } from "./document-kind-meta";
import { documentDateKey } from "./vault-utils";
@@ -167,6 +168,9 @@ export function EpisodeDocumentsCard({ episodeId }: { episodeId: string }) {
{t("documents.episodeCard.upload")}
</Link>
</Button>
<ImportFromSourceButton
link={{ kind: "conditionEpisode", id: episodeId }}
/>
</>
) : null}
{hasMore ? (
@@ -0,0 +1,275 @@
/**
* The document picker's client gates (#1038), pinned via static renders:
*
* - the import button exists only for a person in their own record with a
* usable connection; a delegate, an unset operator list, no connection, or
* a connection whose origin was removed leave no button;
* - its label names the one connected system, or stays generic for two;
* - the settings card renders nothing unless the operator enabled the
* picker, lists the allowed origins, and never puts a token in the markup;
* - every error code the routes emit has its own sentence, and an unknown
* one falls back to the generic sentence rather than a raw key.
*/
import { describe, expect, it, vi } from "vitest";
import { renderToStaticMarkup } from "react-dom/server";
import { QueryClient, QueryClientProvider } from "@tanstack/react-query";
let modules: Record<string, boolean> = { inboundDocuments: true };
vi.mock("@/hooks/use-auth", () => ({
useAuth: () => ({
user: { id: "u1", modules },
isLoading: false,
isAuthenticated: true,
}),
}));
let shared = false;
vi.mock("@/hooks/use-record-capabilities", () => ({
useRecordCapabilities: () => ({
inSharedRecord: shared,
canManageDomain: () => !shared,
}),
}));
vi.mock("@/lib/api/api-fetch", async () => {
const actual = await vi.importActual<typeof import("@/lib/api/api-fetch")>(
"@/lib/api/api-fetch",
);
return { ...actual, apiGet: () => new Promise(() => {}) };
});
import { DocumentSourcesCard } from "@/components/settings/integrations/document-sources-card";
import en from "../../../../../messages/en.json";
import { ApiError } from "@/lib/api/api-fetch";
import {
DOCUMENT_SOURCE_ERROR_CODES,
type DocumentSourcesStatusDto,
} from "@/lib/documents/sources/types";
import { I18nProvider } from "@/lib/i18n/context";
import { queryKeys } from "@/lib/query-keys";
import { ImportFromSourceButton } from "../import-from-source-button";
import { mergeIds } from "../merge-ids";
import { nextChoiceIndex } from "../segmented-choice";
import { errorCodeOf, sourceErrorMessage } from "../source-errors";
const PAPERLESS = {
system: "PAPERLESS" as const,
baseUrl: "http://paperless.lan:8000",
organizationId: null,
hasToken: true as const,
lastVerifiedAt: "2026-09-20T10:00:00.000Z",
originAllowed: true,
};
const PAPRA = {
...PAPERLESS,
system: "PAPRA" as const,
baseUrl: "https://papra.example.com",
organizationId: "org_1",
};
function render(node: React.ReactNode, status?: DocumentSourcesStatusDto) {
const queryClient = new QueryClient({
defaultOptions: { queries: { retry: false } },
});
if (status)
queryClient.setQueryData(queryKeys.documentSourcesStatus(), status);
return renderToStaticMarkup(
<QueryClientProvider client={queryClient}>
<I18nProvider initialLocale="en">{node}</I18nProvider>
</QueryClientProvider>,
);
}
const on = (connections: DocumentSourcesStatusDto["connections"]) => ({
available: true,
allowedOrigins: ["http://paperless.lan:8000", "https://papra.example.com"],
connections,
});
describe("<ImportFromSourceButton>", () => {
it("names the one connected system", () => {
const html = render(<ImportFromSourceButton />, on([PAPERLESS]));
expect(html).toContain('data-slot="document-source-open"');
expect(html).toContain("Import from Paperless-ngx");
});
it("stays generic with two", () => {
const html = render(<ImportFromSourceButton />, on([PAPERLESS, PAPRA]));
expect(html).toContain("Import from archive");
});
it("renders nothing without a usable connection", () => {
expect(render(<ImportFromSourceButton />, on([]))).toBe("");
expect(
render(
<ImportFromSourceButton />,
on([{ ...PAPERLESS, originAllowed: false }]),
),
).toBe("");
expect(
render(<ImportFromSourceButton />, {
available: false,
allowedOrigins: [],
connections: [],
}),
).toBe("");
});
it("renders nothing for a delegate, even with the owner's data cached", () => {
shared = true;
try {
expect(render(<ImportFromSourceButton />, on([PAPERLESS]))).toBe("");
} finally {
shared = false;
}
});
it("renders nothing with the documents module off", () => {
modules = { inboundDocuments: false };
try {
expect(render(<ImportFromSourceButton />, on([PAPERLESS]))).toBe("");
} finally {
modules = { inboundDocuments: true };
}
});
});
describe("<DocumentSourcesCard>", () => {
it("renders nothing when the operator has not enabled the picker", () => {
expect(
render(<DocumentSourcesCard />, {
available: false,
allowedOrigins: [],
connections: [],
}),
).toBe("");
expect(render(<DocumentSourcesCard />)).toBe("");
});
it("renders nothing for a delegate", () => {
shared = true;
try {
expect(render(<DocumentSourcesCard />, on([PAPERLESS]))).toBe("");
} finally {
shared = false;
}
});
it("shows the form, the allowed origins and a saved-token placeholder, never a token", () => {
const html = render(<DocumentSourcesCard />, on([PAPERLESS]));
expect(html).toContain('data-testid="document-sources-card"');
expect(html).toContain("Document archives");
expect(html).toContain(
"Allowed on this server: http://paperless.lan:8000, https://papra.example.com",
);
expect(html).toContain("Saved for this address. Leave empty to keep it.");
expect(html).toContain('value="http://paperless.lan:8000"');
// The password field is empty: the server never sends the token back.
expect(html).toMatch(/id="document-source-token-paperless"[^>]*value=""/);
expect(html).toContain("Disconnect");
});
it("keeps a stored connection visible and removable once the operator list is gone", () => {
const html = render(<DocumentSourcesCard />, {
available: false,
allowedOrigins: [],
connections: [{ ...PAPERLESS, originAllowed: false }],
});
expect(html).toContain('data-testid="document-sources-card"');
expect(html).toContain("no longer allows document archives");
expect(html).toContain("Disconnect");
expect(html).not.toContain("Save and test");
expect(html).not.toContain('id="document-source-token-paperless"');
});
it("keeps a stored connection removable with the documents module off", () => {
modules = { inboundDocuments: false };
try {
const html = render(<DocumentSourcesCard />, on([PAPERLESS]));
expect(html).toContain("Documents are switched off");
expect(html).toContain("Disconnect");
expect(html).not.toContain("Save and test");
} finally {
modules = { inboundDocuments: true };
}
});
it("uses one tab stop for the system switch", () => {
const html = render(<DocumentSourcesCard />, on([PAPERLESS]));
expect(html.match(/role="radio"[^>]*tabindex="0"/gi)?.length).toBe(1);
expect(html.match(/role="radio"[^>]*tabindex="-1"/gi)?.length).toBe(1);
});
it("offers no disconnect or test for a system that is not connected", () => {
const html = render(<DocumentSourcesCard />, on([]));
expect(html).toContain("Not connected");
expect(html).not.toContain("Disconnect");
expect(html).toContain("Save and test");
});
});
describe("source error sentences", () => {
const t = (key: string) => {
let node: unknown = en;
for (const part of key.split(".")) {
node = (node as Record<string, unknown> | undefined)?.[part];
}
return typeof node === "string" ? node : key;
};
const generic = t("documents.sourcePicker.errors.generic");
it("has its own sentence for every code the routes emit", () => {
const codes = [
...DOCUMENT_SOURCE_ERROR_CODES,
"documents.sources.browserOnly",
"documents.sources.invalidAddress",
"documents.sources.organizationRequired",
"documents.sources.tokenRequired",
"documents.inbound.fileTooLarge",
"documents.inbound.quotaExceeded",
"documents.inbound.fileType",
"documents.inbound.rateLimited",
"documents.inbound.uploadBusy",
"module.disabled",
];
const sentences = new Set<string>();
for (const code of codes) {
const sentence = sourceErrorMessage(t, code);
expect(sentence, code).not.toBe(generic);
expect(sentence, code).not.toContain("documents.sourcePicker");
sentences.add(sentence);
}
expect(sentences.size).toBe(codes.length);
expect(sourceErrorMessage(t, "something.else")).toBe(generic);
expect(sourceErrorMessage(t, undefined)).toBe(generic);
});
it("reads the code off a failed request", () => {
expect(
errorCodeOf(
new ApiError("x", 502, { errorCode: "documents.sources.authRefused" }),
),
).toBe("documents.sources.authRefused");
expect(errorCodeOf(new Error("x"))).toBeUndefined();
});
});
describe("nextChoiceIndex", () => {
it("moves with the arrows, wraps, and jumps with Home and End", () => {
expect(nextChoiceIndex("ArrowRight", 0, 2)).toBe(1);
expect(nextChoiceIndex("ArrowRight", 1, 2)).toBe(0);
expect(nextChoiceIndex("ArrowLeft", 0, 2)).toBe(1);
expect(nextChoiceIndex("ArrowDown", 0, 3)).toBe(1);
expect(nextChoiceIndex("ArrowUp", 0, 3)).toBe(2);
expect(nextChoiceIndex("Home", 2, 3)).toBe(0);
expect(nextChoiceIndex("End", 0, 3)).toBe(2);
expect(nextChoiceIndex("Enter", 0, 3)).toBeNull();
});
});
describe("mergeIds", () => {
it("adds imported ids to a form's selection once, in order", () => {
expect(mergeIds(["a", "b"], ["b", "c", "c"])).toEqual(["a", "b", "c"]);
});
});
@@ -0,0 +1,764 @@
"use client";
/**
* The document picker (#1038): search a connected Paperless-ngx or Papra by
* document name, narrow by tag and date, tick the documents to bring over,
* import them.
*
* Name first, because that is how the person looks for a letter: the search
* field opens focused and debounces into the source's own name search. Tag and
* date are filters beside it, not the entry point. Each result says what
* HealthLog already holds under that source key: already imported, deleted
* here (not offered again: a deletion was a decision), or new.
*
* The import runs one document per request, in order, so every row can show
* its own progress and result, and one failure does not hide the others. A
* refusal that will answer every following document the same way (the hourly
* upload allowance, a full vault, a refused token) stops the run and says so.
*
* Where it was opened from decides what an import does besides storing:
*
* - `link`: the record already exists (a condition's page), so the server
* files the document against it as part of the import;
* - `onImported`: a form that owns its selection (a visit, a vaccination)
* gets the new ids and adds them itself. Linking on the server there would
* be undone by the form's own save, which replaces its links.
*
* With either, a document already in HealthLog can be picked too: it is not
* stored twice, it is linked or handed to the form.
*/
import {
useInfiniteQuery,
useQuery,
useQueryClient,
} from "@tanstack/react-query";
import {
Check,
CircleAlert,
CircleCheck,
Loader2,
Search,
SearchX,
X,
} from "lucide-react";
import { useEffect, useMemo, useRef, useState } from "react";
import { Badge } from "@/components/ui/badge";
import { Button } from "@/components/ui/button";
import { EmptyState } from "@/components/ui/empty-state";
import { DateField } from "@/components/ui/date-field";
import { Input } from "@/components/ui/input";
import { Label } from "@/components/ui/label";
import { NativeSelect } from "@/components/ui/native-select";
import { QueryErrorRow } from "@/components/ui/query-error-row";
import { ResponsiveSheet } from "@/components/ui/responsive-sheet";
import { Skeleton } from "@/components/ui/skeleton";
import { TagChip } from "@/components/ui/tag-chip";
import { apiGet, apiPost } from "@/lib/api/api-fetch";
import {
DOCUMENT_PICKER_MAX_SELECTION,
slugForSystem,
type DocumentImportOutcome,
type DocumentPickerLinkKind,
type DocumentPickerSystem,
type DocumentSourceImportDto,
type DocumentSourceResultDto,
type DocumentSourceSearchDto,
type DocumentSourceTagDto,
} from "@/lib/documents/sources/types";
import { useFormatters, useTranslations } from "@/lib/i18n/context";
import { invalidateKeys, queryKeys } from "@/lib/query-keys";
import { cn } from "@/lib/utils";
import type { InboundDocumentKindValue } from "@/lib/validations/inbound-documents";
import { WrittenOutcomeLine } from "@/components/outcome/written-outcome-line";
import type { WrittenOutcome } from "@/lib/outcome/written-outcome";
import { formatBytes } from "../vault-utils";
import {
errorCodeOf,
isStoppingError,
sourceErrorMessage,
} from "./source-errors";
import { SegmentedChoice } from "./segmented-choice";
import { systemName } from "./use-document-sources";
export interface DocumentPickerLink {
kind: DocumentPickerLinkKind;
id: string;
}
export interface DocumentSourcePickerProps {
open: boolean;
onOpenChange: (open: boolean) => void;
/** Connected systems, in the order the switch shows them. */
systems: DocumentPickerSystem[];
/** File the imported documents against this existing record. */
link?: DocumentPickerLink;
/** Document type for everything imported (default OTHER). */
kind?: InboundDocumentKindValue;
/** Hand the vault ids to a form that links them on its own save. */
onImported?: (documentIds: string[]) => void;
}
type RowRun =
| { state: "waiting" }
| { state: "working" }
| { state: DocumentImportOutcome; linked: boolean }
| { state: "failed"; message: string };
const SEARCH_DEBOUNCE_MS = 300;
const IMPORT_TIMEOUT_MS = 120_000;
export function DocumentSourcePicker({
open,
onOpenChange,
systems,
link,
kind,
onImported,
}: DocumentSourcePickerProps) {
const { t, locale } = useTranslations();
const format = useFormatters();
const queryClient = useQueryClient();
const [system, setSystem] = useState<DocumentPickerSystem | null>(null);
const active =
system && systems.includes(system) ? system : (systems[0] ?? null);
const slug = active ? slugForSystem(active) : null;
const [searchDraft, setSearchDraft] = useState("");
const [q, setQ] = useState("");
const [tag, setTag] = useState("");
const [from, setFrom] = useState("");
const [to, setTo] = useState("");
const [selected, setSelected] = useState<Map<string, string>>(new Map());
const [runs, setRuns] = useState<Map<string, RowRun>>(new Map());
const [running, setRunning] = useState(false);
const [finished, setFinished] = useState(false);
const [stopMessage, setStopMessage] = useState<string | null>(null);
const [atMax, setAtMax] = useState(false);
/** The latest per-row result, for screen readers (aria-live below). */
const [announcement, setAnnouncement] = useState("");
// A new search or filter starts over: a selection made in one result list
// must not ride along unseen into another, and a finished run's result
// lines belong to the list they were made in.
const startOver = () => {
setSelected(new Map());
setRuns(new Map());
setFinished(false);
setStopMessage(null);
setAtMax(false);
};
const debounceRef = useRef<ReturnType<typeof setTimeout> | null>(null);
useEffect(
() => () => {
if (debounceRef.current) clearTimeout(debounceRef.current);
},
[],
);
const onSearchChange = (value: string) => {
setSearchDraft(value);
if (debounceRef.current) clearTimeout(debounceRef.current);
debounceRef.current = setTimeout(() => {
const next = value.trim();
if (next !== q) {
startOver();
setQ(next);
}
}, SEARCH_DEBOUNCE_MS);
};
const filterSetter =
(set: (value: string) => void) =>
(value: string): void => {
startOver();
set(value);
};
const reset = () => {
setSearchDraft("");
setQ("");
setTag("");
setFrom("");
setTo("");
startOver();
};
const switchSystem = (next: DocumentPickerSystem) => {
if (next === active || running) return;
reset();
setSystem(next);
};
const tags = useQuery({
queryKey: queryKeys.documentSourceTags(slug ?? ""),
enabled: open && slug !== null,
queryFn: () =>
apiGet<{ tags: DocumentSourceTagDto[] }>(
`/api/documents/sources/${slug}/tags`,
),
staleTime: 5 * 60_000,
});
const facets = {
q,
...(tag ? { tag } : {}),
...(from ? { from } : {}),
...(to ? { to } : {}),
};
const rangeInvalid = Boolean(from && to && from > to);
const search = useInfiniteQuery({
queryKey: queryKeys.documentSourceSearch(slug ?? "", facets),
enabled: open && slug !== null && !rangeInvalid,
initialPageParam: 1,
queryFn: ({ pageParam }) => {
const sp = new URLSearchParams({ page: String(pageParam) });
if (q) sp.set("q", q);
if (tag) sp.set("tag", tag);
if (from) sp.set("from", from);
if (to) sp.set("to", to);
return apiGet<DocumentSourceSearchDto>(
`/api/documents/sources/${slug}/search?${sp.toString()}`,
);
},
getNextPageParam: (last) => (last.hasMore ? last.page + 1 : undefined),
staleTime: 30_000,
});
const results = useMemo(() => {
const seen = new Set<string>();
const out: DocumentSourceResultDto[] = [];
for (const page of search.data?.pages ?? []) {
for (const row of page.results) {
if (seen.has(row.sourceId)) continue;
seen.add(row.sourceId);
out.push(row);
}
}
return out;
}, [search.data]);
// An already imported document is worth picking only when the pick does
// something with it: link it, or hand it to the form.
const canTakeImported = Boolean(link || onImported);
const selectable = (row: DocumentSourceResultDto) =>
row.state === "new" || (row.state === "imported" && canTakeImported);
const toggle = (row: DocumentSourceResultDto) => {
if (running || finished || !selectable(row)) return;
const next = new Map(selected);
if (next.has(row.sourceId)) {
next.delete(row.sourceId);
setAtMax(false);
} else if (next.size < DOCUMENT_PICKER_MAX_SELECTION) {
next.set(row.sourceId, row.title);
} else {
// Said rather than silently ignored: the tick that did not appear.
setAtMax(true);
return;
}
setSelected(next);
};
async function runImport() {
if (!slug || selected.size === 0) return;
setRunning(true);
setStopMessage(null);
const order = [...selected.keys()];
setRuns(new Map(order.map((id) => [id, { state: "waiting" }])));
const handed: string[] = [];
const mark = (id: string, run: RowRun) => {
setRuns((prev) => new Map(prev).set(id, run));
const title = selected.get(id) ?? id;
const said = runLabel(t, run);
if (said) setAnnouncement(`${title}: ${said}`);
};
for (const sourceId of order) {
mark(sourceId, { state: "working" });
try {
const result = await apiPost<DocumentSourceImportDto>(
`/api/documents/sources/${slug}/import`,
{
sourceId,
...(kind ? { kind } : {}),
...(link ? { link } : {}),
},
// The server may take up to a minute to fetch a large file from
// the source; the default 15 s client timeout would abandon it.
{ signal: AbortSignal.timeout(IMPORT_TIMEOUT_MS) },
);
mark(sourceId, { state: result.outcome, linked: result.linked });
if (result.outcome !== "deleted" && result.documentId) {
handed.push(result.documentId);
}
} catch (err) {
const code = errorCodeOf(err);
const message = sourceErrorMessage(t, code);
mark(sourceId, { state: "failed", message });
if (isStoppingError(code)) {
setStopMessage(message);
break;
}
}
}
setRunning(false);
setFinished(true);
setSelected(new Map());
// The vault, its pickers and the record's own lists read documents
// through the `["documents"]` prefix; the picker's states through its own.
void invalidateKeys(queryClient, [
queryKeys.documents(),
queryKeys.documentSources(),
]);
if (handed.length > 0) onImported?.(handed);
}
const counts = useMemo(() => {
let imported = 0;
let present = 0;
let failed = 0;
for (const run of runs.values()) {
if (run.state === "imported") imported += 1;
else if (run.state === "duplicate" || run.state === "deleted")
present += 1;
else if (run.state === "failed") failed += 1;
}
return { imported, present, failed };
}, [runs]);
const name = active ? systemName(active) : "";
const hasFilters = Boolean(tag || from || to);
const footer = (
<div className="flex w-full items-center justify-between gap-3">
<p
className="text-muted-foreground min-w-0 text-xs"
data-slot="document-source-selection"
>
{finished
? null
: running
? t("documents.sourcePicker.importing")
: t("documents.sourcePicker.selectedCount", {
count: selected.size,
max: DOCUMENT_PICKER_MAX_SELECTION,
})}
</p>
{finished ? (
<div className="flex shrink-0 items-center gap-2">
<Button
type="button"
variant="outline"
size="sm"
className="min-h-11 sm:min-h-9"
onClick={startOver}
data-slot="document-source-pick-more"
>
{t("documents.sourcePicker.pickMore")}
</Button>
<Button
type="button"
size="sm"
className="min-h-11 sm:min-h-9"
onClick={() => onOpenChange(false)}
>
{t("documents.sourcePicker.done")}
</Button>
</div>
) : (
<Button
type="button"
size="sm"
className="min-h-11 shrink-0 sm:min-h-9"
disabled={running || selected.size === 0}
onClick={() => void runImport()}
data-slot="document-source-import"
>
{running ? (
<Loader2
className="size-4 animate-spin motion-reduce:animate-none"
aria-hidden
/>
) : null}
{t("documents.sourcePicker.importAction", { count: selected.size })}
</Button>
)}
</div>
);
return (
<ResponsiveSheet
open={open}
onOpenChange={(next) => {
// A run in progress finishes before the sheet can close; its rows
// are the only place the outcome is shown.
if (!next && running) return;
if (!next) reset();
onOpenChange(next);
}}
title={t("documents.sourcePicker.title", { name })}
description={t("documents.sourcePicker.description")}
contentWidth="lg"
footer={footer}
>
<div className="space-y-3" data-slot="document-source-picker">
{systems.length > 1 && active ? (
<SegmentedChoice
options={systems}
value={active}
onChange={switchSystem}
label={t("documents.sourcePicker.sourceLabel")}
disabled={running}
renderOption={systemName}
/>
) : null}
<div className="space-y-1.5">
<Label htmlFor="document-source-search">
{t("documents.sourcePicker.searchLabel")}
</Label>
<div className="relative">
<Search
className="text-muted-foreground pointer-events-none absolute top-1/2 left-3 size-4 -translate-y-1/2"
aria-hidden
/>
<Input
id="document-source-search"
type="search"
autoFocus
value={searchDraft}
onChange={(e) => onSearchChange(e.target.value)}
placeholder={t("documents.sourcePicker.searchPlaceholder")}
className="pl-9"
maxLength={200}
autoComplete="off"
data-slot="document-source-search"
/>
</div>
</div>
<div className="grid grid-cols-2 gap-3 sm:grid-cols-3">
<div className="col-span-2 space-y-1.5 sm:col-span-1">
<Label htmlFor="document-source-tag">
{t("documents.sourcePicker.tagLabel")}
</Label>
<NativeSelect
id="document-source-tag"
value={tag}
onChange={(e) => filterSetter(setTag)(e.target.value)}
disabled={tags.isPending && !tags.isError}
>
<option value="">{t("documents.sourcePicker.tagAny")}</option>
{(tags.data?.tags ?? []).map((option) => (
<option key={option.id} value={option.id}>
{option.name}
</option>
))}
</NativeSelect>
</div>
<div className="space-y-1.5">
<Label htmlFor="document-source-from">
{t("documents.sourcePicker.fromLabel")}
</Label>
<DateField
id="document-source-from"
value={from}
onChange={filterSetter(setFrom)}
max={to || undefined}
/>
</div>
<div className="space-y-1.5">
<Label htmlFor="document-source-to">
{t("documents.sourcePicker.toLabel")}
</Label>
<DateField
id="document-source-to"
value={to}
onChange={filterSetter(setTo)}
min={from || undefined}
/>
</div>
</div>
{hasFilters ? (
<button
type="button"
onClick={() => {
startOver();
setTag("");
setFrom("");
setTo("");
}}
className="text-muted-foreground hover:text-foreground focus-visible:ring-ring/50 inline-flex min-h-8 items-center gap-1 rounded-sm text-xs underline-offset-2 hover:underline focus-visible:ring-[3px] focus-visible:outline-none"
>
<X className="size-3.5" aria-hidden />
{t("documents.sourcePicker.clearFilters")}
</button>
) : null}
{finished ? (
<div className="space-y-1">
<WrittenOutcomeLine
outcome={runOutcome(counts)}
message={t("documents.sourcePicker.summary", counts)}
testId="document-source-summary"
/>
<p className="text-muted-foreground text-xs">
{t("documents.sourcePicker.finishedHint")}
</p>
</div>
) : null}
{atMax ? (
<p
role="status"
className="text-muted-foreground text-xs"
data-slot="document-source-at-max"
>
{t("documents.sourcePicker.maxReached", {
max: DOCUMENT_PICKER_MAX_SELECTION,
})}
</p>
) : null}
{/* Each row's result as it lands, for a screen reader. */}
<div aria-live="polite" className="sr-only">
{announcement}
</div>
{stopMessage ? (
<p
role="alert"
className="text-destructive text-sm"
data-slot="document-source-stopped"
>
{t("documents.sourcePicker.stopped", { reason: stopMessage })}
</p>
) : null}
{rangeInvalid ? (
<p role="alert" className="text-destructive text-sm">
{t("documents.sourcePicker.rangeInvalid")}
</p>
) : search.isPending ? (
<div className="space-y-2" data-slot="document-source-loading">
{Array.from({ length: 4 }, (_, i) => (
<Skeleton key={i} className="h-14 w-full rounded-lg" />
))}
</div>
) : search.isError ? (
<QueryErrorRow
message={sourceErrorMessage(t, errorCodeOf(search.error))}
onRetry={() => void search.refetch()}
slot="document-source-error"
/>
) : results.length === 0 ? (
<EmptyState
variant="plain"
size="compact"
icon={<SearchX className="size-6" aria-hidden />}
title={
q || hasFilters
? t("documents.sourcePicker.noMatch")
: t("documents.sourcePicker.empty", { name })
}
description={
q || hasFilters
? t("documents.sourcePicker.noMatchHint")
: undefined
}
/>
) : (
<ul
className="max-h-[45vh] space-y-1.5 overflow-y-auto overscroll-contain"
data-slot="document-source-results"
>
{results.map((row) => {
const on = selected.has(row.sourceId);
const run = runs.get(row.sourceId);
const allowed = selectable(row);
const meta = [
row.date ? format.date(`${row.date}T12:00:00.000Z`) : null,
row.sizeBytes !== null
? formatBytes(row.sizeBytes, locale)
: null,
]
.filter(Boolean)
.join(" · ");
return (
<li key={row.sourceId}>
<button
type="button"
aria-pressed={on}
aria-disabled={!allowed || running || finished}
onClick={() => toggle(row)}
data-slot="document-source-result"
data-state={row.state}
className={cn(
"border-border flex min-h-11 w-full items-start gap-3 rounded-lg border p-3 text-left",
"focus-visible:ring-ring/50 focus-visible:ring-[3px] focus-visible:outline-none",
allowed && !running && !finished && "hover:bg-muted/50",
on && "border-primary/40 bg-primary/5",
// A row that just ran keeps full contrast: its result is
// the thing to read.
// After a run, the rows it did not touch step back:
// the ones with a result are what to read now.
((!allowed && !run) || (finished && !run)) &&
"opacity-50",
)}
>
<Check
className={cn(
"text-primary mt-0.5 size-4 shrink-0",
on ? "opacity-100" : "opacity-0",
)}
aria-hidden
/>
<span className="min-w-0 flex-1 space-y-1">
<span className="block truncate text-sm font-medium">
{row.title}
</span>
{meta ? (
<span className="text-muted-foreground block text-xs">
{meta}
</span>
) : null}
{row.tags.length > 0 ? (
<span className="flex flex-wrap gap-1">
{row.tags.slice(0, 6).map((name) => (
<TagChip key={name}>{name}</TagChip>
))}
</span>
) : null}
{run ? <RunLine run={run} /> : null}
</span>
{row.state !== "new" && !run ? (
<Badge variant="outline" className="shrink-0">
{row.state === "imported"
? t("documents.sourcePicker.stateImported")
: t("documents.sourcePicker.stateDeleted")}
</Badge>
) : null}
</button>
</li>
);
})}
</ul>
)}
{search.hasNextPage && !search.isError ? (
<Button
type="button"
variant="outline"
size="sm"
className="min-h-11 w-full sm:min-h-9"
disabled={search.isFetchingNextPage}
onClick={() => void search.fetchNextPage()}
>
{search.isFetchingNextPage ? (
<Loader2
className="size-4 animate-spin motion-reduce:animate-none"
aria-hidden
/>
) : null}
{t("documents.sourcePicker.loadMore")}
</Button>
) : null}
</div>
</ResponsiveSheet>
);
}
/** How the run went, in the shared outcome vocabulary. */
function runOutcome(counts: {
imported: number;
present: number;
failed: number;
}): WrittenOutcome {
const held = counts.imported + counts.present;
if (counts.failed === 0) return held > 0 ? "success" : "empty";
return held > 0 ? "partial" : "failed";
}
/** A row's result in words, or null while it has none yet. */
function runLabel(
t: (key: string, params?: Record<string, string | number>) => string,
run: RowRun,
): string | null {
switch (run.state) {
case "waiting":
case "working":
return null;
case "failed":
return run.message;
case "deleted":
return t("documents.sourcePicker.result.deleted");
case "imported":
return run.linked
? t("documents.sourcePicker.result.importedLinked")
: t("documents.sourcePicker.result.imported");
case "duplicate":
return run.linked
? t("documents.sourcePicker.result.duplicateLinked")
: t("documents.sourcePicker.result.duplicate");
}
}
/** One row's progress or result, under its title. */
function RunLine({ run }: { run: RowRun }) {
const { t } = useTranslations();
switch (run.state) {
case "waiting":
return (
<span className="text-muted-foreground block text-xs">
{t("documents.sourcePicker.result.waiting")}
</span>
);
case "working":
return (
<span className="text-muted-foreground flex items-center gap-1.5 text-xs">
<Loader2
className="size-3.5 animate-spin motion-reduce:animate-none"
aria-hidden
/>
{t("documents.sourcePicker.result.working")}
</span>
);
case "failed":
return (
<span
className="text-destructive flex items-start gap-1.5 text-xs"
data-slot="document-source-run-failed"
>
<CircleAlert className="mt-px size-3.5 shrink-0" aria-hidden />
{run.message}
</span>
);
case "deleted":
return (
<span className="text-muted-foreground block text-xs">
{t("documents.sourcePicker.result.deleted")}
</span>
);
case "imported":
case "duplicate":
return (
<span
className="text-success flex items-center gap-1.5 text-xs"
data-slot="document-source-run-done"
>
<CircleCheck className="size-3.5 shrink-0" aria-hidden />
{run.state === "imported"
? run.linked
? t("documents.sourcePicker.result.importedLinked")
: t("documents.sourcePicker.result.imported")
: run.linked
? t("documents.sourcePicker.result.duplicateLinked")
: t("documents.sourcePicker.result.duplicate")}
</span>
);
}
}
@@ -0,0 +1,73 @@
"use client";
/**
* "Import from Paperless-ngx / Papra": the button that opens the document
* picker (#1038), for every place a document can be added or linked.
*
* Renders nothing unless the person, in their own record, has at least one
* usable connection: no connection, a delegate or guardian, the module off, or
* the operator's list unset all leave no trace, because a button that can only
* lead to a refusal advertises a feature that is not there. Connecting happens
* in Settings, not here.
*/
import { Import } from "lucide-react";
import { useState } from "react";
import { Button } from "@/components/ui/button";
import { useTranslations } from "@/lib/i18n/context";
import type { InboundDocumentKindValue } from "@/lib/validations/inbound-documents";
import {
DocumentSourcePicker,
type DocumentPickerLink,
} from "./document-source-picker";
import { systemName, useDocumentSourcesStatus } from "./use-document-sources";
export function ImportFromSourceButton({
link,
kind,
onImported,
className,
}: {
link?: DocumentPickerLink;
kind?: InboundDocumentKindValue;
onImported?: (documentIds: string[]) => void;
className?: string;
}) {
const { t } = useTranslations();
const { connected } = useDocumentSourcesStatus();
const [open, setOpen] = useState(false);
if (connected.length === 0) return null;
const label =
connected.length === 1
? t("documents.sourcePicker.openNamed", {
name: systemName(connected[0]),
})
: t("documents.sourcePicker.open");
return (
<>
<Button
type="button"
variant="outline"
size="sm"
className={className}
onClick={() => setOpen(true)}
data-slot="document-source-open"
>
<Import className="size-4" aria-hidden />
{label}
</Button>
<DocumentSourcePicker
open={open}
onOpenChange={setOpen}
systems={connected}
link={link}
kind={kind}
onImported={onImported}
/>
</>
);
}
@@ -0,0 +1,6 @@
/** Add imported ids to a form's selection, in order, never twice. */
export function mergeIds(selected: string[], added: string[]): string[] {
const merged = [...selected];
for (const id of added) if (!merged.includes(id)) merged.push(id);
return merged;
}
@@ -0,0 +1,110 @@
"use client";
/**
* A small segmented radio group (Paperless-ngx | Papra) with the keyboard
* contract of a radio group: one tab stop (the checked option), arrow keys
* move and select, Home and End jump to the ends. Shared by the document
* picker's source switch and the settings card's system switch.
*/
import { useRef, type KeyboardEvent, type ReactNode } from "react";
import { cn } from "@/lib/utils";
/**
* The option an arrow, Home or End key moves to from `index`, wrapping at the
* ends; null for any other key.
*/
export function nextChoiceIndex(
key: string,
index: number,
count: number,
): number | null {
if (count <= 0) return null;
switch (key) {
case "ArrowRight":
case "ArrowDown":
return (index + 1) % count;
case "ArrowLeft":
case "ArrowUp":
return (index - 1 + count) % count;
case "Home":
return 0;
case "End":
return count - 1;
default:
return null;
}
}
export function SegmentedChoice<T extends string>({
options,
value,
onChange,
label,
disabled = false,
renderOption,
slot,
className,
}: {
options: readonly T[];
value: T;
onChange: (value: T) => void;
/** Accessible name of the group. */
label: string;
disabled?: boolean;
renderOption: (option: T) => ReactNode;
slot?: string;
className?: string;
}) {
const refs = useRef<Array<HTMLButtonElement | null>>([]);
const onKeyDown = (event: KeyboardEvent<HTMLButtonElement>, i: number) => {
const next = nextChoiceIndex(event.key, i, options.length);
if (next === null) return;
event.preventDefault();
if (disabled) return;
onChange(options[next]);
refs.current[next]?.focus();
};
return (
<div
role="radiogroup"
aria-label={label}
aria-disabled={disabled || undefined}
className={cn(
"bg-muted inline-flex self-start rounded-md p-1",
className,
)}
>
{options.map((option, i) => {
const checked = option === value;
return (
<button
key={option}
ref={(el) => {
refs.current[i] = el;
}}
type="button"
role="radio"
aria-checked={checked}
tabIndex={checked ? 0 : -1}
disabled={disabled}
onClick={() => onChange(option)}
onKeyDown={(event) => onKeyDown(event, i)}
data-slot={slot}
className={cn(
"flex min-h-9 items-center gap-1.5 rounded-sm px-3 text-sm font-medium",
"focus-visible:ring-ring/50 focus-visible:ring-[3px] focus-visible:outline-none",
checked
? "bg-background text-foreground shadow-xs"
: "text-muted-foreground hover:text-foreground",
)}
>
{renderOption(option)}
</button>
);
})}
</div>
);
}
@@ -0,0 +1,84 @@
/**
* The document picker's error codes (#1038), as sentences. One place for the
* sheet and the settings card, so the same refusal reads the same on both.
* Every branch names its key literally so the i18n coverage guard can see it.
*/
import { ApiError } from "@/lib/api/api-fetch";
type T = (key: string, params?: Record<string, string | number>) => string;
/** The `meta.errorCode` a failed request carried, if any. */
export function errorCodeOf(err: unknown): string | undefined {
if (err instanceof ApiError) {
const code = err.meta?.errorCode;
if (typeof code === "string") return code;
}
return undefined;
}
/** Is this a refusal that will not change on the next document? */
export function isStoppingError(code: string | undefined): boolean {
return (
code === "documents.inbound.rateLimited" ||
code === "documents.inbound.quotaExceeded" ||
code === "documents.sources.authRefused" ||
code === "documents.sources.permissionMissing" ||
code === "documents.sources.unreachable" ||
code === "documents.sources.originNotAllowed" ||
code === "documents.sources.notConnected" ||
code === "documents.sources.unavailable" ||
code === "documents.sources.browserOnly" ||
code === "documents.sources.linkTargetNotFound"
);
}
export function sourceErrorMessage(t: T, code: string | undefined): string {
switch (code) {
case "documents.sources.unavailable":
return t("documents.sourcePicker.errors.unavailable");
case "documents.sources.notConnected":
return t("documents.sourcePicker.errors.notConnected");
case "documents.sources.originNotAllowed":
return t("documents.sourcePicker.errors.originNotAllowed");
case "documents.sources.unreachable":
return t("documents.sourcePicker.errors.unreachable");
case "documents.sources.redirected":
return t("documents.sourcePicker.errors.redirected");
case "documents.sources.authRefused":
return t("documents.sourcePicker.errors.authRefused");
case "documents.sources.permissionMissing":
return t("documents.sourcePicker.errors.permissionMissing");
case "documents.sources.versionTooOld":
return t("documents.sourcePicker.errors.versionTooOld");
case "documents.sources.badResponse":
return t("documents.sourcePicker.errors.badResponse");
case "documents.sources.notFound":
return t("documents.sourcePicker.errors.notFound");
case "documents.sources.linkTargetNotFound":
return t("documents.sourcePicker.errors.linkTargetNotFound");
case "documents.sources.rateLimited":
return t("documents.sourcePicker.errors.rateLimited");
case "documents.sources.browserOnly":
return t("documents.sourcePicker.errors.browserOnly");
case "documents.sources.invalidAddress":
return t("documents.sourcePicker.errors.invalidAddress");
case "documents.sources.organizationRequired":
return t("documents.sourcePicker.errors.organizationRequired");
case "documents.sources.tokenRequired":
return t("documents.sourcePicker.errors.tokenRequired");
case "documents.inbound.fileTooLarge":
return t("documents.sourcePicker.errors.fileTooLarge");
case "documents.inbound.quotaExceeded":
return t("documents.sourcePicker.errors.quotaExceeded");
case "documents.inbound.fileType":
return t("documents.sourcePicker.errors.fileType");
case "documents.inbound.rateLimited":
return t("documents.sourcePicker.errors.uploadLimit");
case "documents.inbound.uploadBusy":
return t("documents.sourcePicker.errors.uploadBusy");
case "module.disabled":
return t("documents.sourcePicker.errors.moduleDisabled");
default:
return t("documents.sourcePicker.errors.generic");
}
}
@@ -0,0 +1,58 @@
"use client";
/**
* The document picker's availability for the person on this screen (#1038).
*
* Reads `GET /api/documents/sources` only in the person's own record: a
* delegate or guardian never asks (the route would refuse them anyway), so
* the import buttons and the settings card render nothing for them.
* `connected` is the list of systems with a usable connection (the documents
* module on, the operator's list set, the origin still on it), in a stable
* order; the import buttons read only that.
*/
import { useQuery } from "@tanstack/react-query";
import { useAuth } from "@/hooks/use-auth";
import { useRecordCapabilities } from "@/hooks/use-record-capabilities";
import { apiGet } from "@/lib/api/api-fetch";
import type {
DocumentPickerSystem,
DocumentSourcesStatusDto,
} from "@/lib/documents/sources/types";
import { queryKeys } from "@/lib/query-keys";
export function useDocumentSourcesStatus() {
const { user } = useAuth();
const { canManageDomain, inSharedRecord } = useRecordCapabilities();
// The status read touches only the person's own rows and never the
// network, so it runs in one's own record whatever the module says: a
// connection stored before Documents was switched off, or before the
// operator removed the list, must stay visible so it can be deleted.
const ownRecord =
user !== null &&
user !== undefined &&
!inSharedRecord &&
canManageDomain("documents");
const eligible = ownRecord && user?.modules?.inboundDocuments === true;
const query = useQuery({
queryKey: queryKeys.documentSourcesStatus(),
enabled: ownRecord,
queryFn: () => apiGet<DocumentSourcesStatusDto>("/api/documents/sources"),
staleTime: 60_000,
});
const connected: DocumentPickerSystem[] =
eligible && query.data?.available
? query.data.connections
.filter((c) => c.originAllowed)
.map((c) => c.system)
: [];
return { ownRecord, eligible, query, connected };
}
/** Display name of a system; product names, not translated. */
export function systemName(system: DocumentPickerSystem): string {
return system === "PAPERLESS" ? "Paperless-ngx" : "Papra";
}
@@ -32,6 +32,8 @@ import {
type EntityLinkOption,
} from "@/components/links/entity-link-picker";
import { useVaultDocumentOptions } from "@/components/links/vault-document-options";
import { ImportFromSourceButton } from "@/components/documents/sources/import-from-source-button";
import { mergeIds } from "@/components/documents/sources/merge-ids";
/** Fetch cap for the picker — the grouped sheet needs enough rows to group. */
export const PICKER_FETCH_LIMIT = 200;
@@ -152,6 +154,17 @@ export function EncounterLinkPickers({
options={documents.options}
searchPlaceholder={t("links.picker.searchPlaceholder")}
emptyLabel={t("encounters.form.linkNothingToOffer")}
extraAction={
// The form owns its selection and replaces the visit's links on
// save, so imported documents join the selection rather than
// being linked on the server.
<ImportFromSourceButton
className="min-h-9"
onImported={(ids) =>
onChange({ documentIds: mergeIds(documentIds, ids) })
}
/>
}
/>
) : null}
+27 -13
View File
@@ -31,7 +31,7 @@
* carries the option's date, so two doses of one vaccine read as two
* different records.
*/
import { useMemo, useState } from "react";
import { useMemo, useState, type ReactNode } from "react";
import Link from "next/link";
import { Check, Plus, Search, X } from "lucide-react";
import type { LucideIcon } from "lucide-react";
@@ -164,6 +164,7 @@ export function EntityLinkPicker({
error = false,
errorLabel,
onRetry,
extraAction,
}: {
icon: LucideIcon;
title: string;
@@ -183,6 +184,13 @@ export function EntityLinkPicker({
error?: boolean;
errorLabel?: string;
onRetry?: () => void;
/**
* A second way to add, beside the add button: the documents pickers put
* "Import from Paperless-ngx / Papra" here. Shown even when there is
* nothing to link yet, because an empty vault is exactly when importing is
* the way forward. The element renders nothing when it has nothing to offer.
*/
extraAction?: ReactNode;
}) {
const { t } = useTranslations();
const [open, setOpen] = useState(false);
@@ -238,7 +246,10 @@ export function EntityLinkPicker({
slot={`${slot}-error`}
/>
) : options.length === 0 ? (
<p className="text-muted-foreground text-xs">{emptyLabel}</p>
<div className="space-y-2">
<p className="text-muted-foreground text-xs">{emptyLabel}</p>
{extraAction}
</div>
) : (
<div className="space-y-2">
{selectedChips.length > 0 ? (
@@ -312,17 +323,20 @@ export function EntityLinkPicker({
</p>
) : null}
<Button
type="button"
variant="outline"
size="sm"
className="min-h-9"
data-slot={`${slot}-add`}
onClick={() => setOpen(true)}
>
<Plus className="size-4" aria-hidden />
{t("links.picker.add")}
</Button>
<div className="flex flex-wrap items-center gap-2">
<Button
type="button"
variant="outline"
size="sm"
className="min-h-9"
data-slot={`${slot}-add`}
onClick={() => setOpen(true)}
>
<Plus className="size-4" aria-hidden />
{t("links.picker.add")}
</Button>
{extraAction}
</div>
</div>
)}
@@ -37,6 +37,7 @@ import { ConnectionsPanel } from "@/components/settings/integrations/connections
import { SectionHeading } from "@/components/ui/section-heading";
import { NotificationChannelsPanel } from "@/components/settings/integrations/notification-channels-panel";
import { NutrientIntakeCard } from "@/components/settings/nutrient-intake-card";
import { DocumentSourcesCard } from "@/components/settings/integrations/document-sources-card";
import { Button } from "@/components/ui/button";
export {
@@ -77,6 +78,11 @@ export function IntegrationsSection({
under the same Connections group rather than a new labelled
section that could paint an orphaned heading over nothing. */}
<NutrientIntakeCard />
{/* v1.39.3 (#1038) — Paperless-ngx / Papra for the document picker.
Renders nothing unless the operator listed an origin in
DOCUMENT_SOURCE_ORIGINS and the person is in their own record. */}
<DocumentSourcesCard />
</section>
{/* Delivery channels ("where it's delivered"). The existing channels
@@ -0,0 +1,453 @@
"use client";
/**
* Settings → Integrations → Document archives (#1038): connect Paperless-ngx or
* Papra so the document picker can search it.
*
* Renders nothing unless the operator listed at least one origin in
* `DOCUMENT_SOURCE_ORIGINS`, the documents module is on, and the person is in
* their own record: a delegate never manages the owner's connection, and a
* card that can only be refused advertises a feature that is not there.
*
* One form per system behind a switch. Save tests the connection before it is
* stored and says why when it fails; the token field never shows the stored
* token (the server never returns it), only that one is saved, and leaving it
* empty keeps it. Results sit above the action row; nothing follows it
* (UI standards §12).
*/
import { useMutation, useQueryClient } from "@tanstack/react-query";
import { Archive, Check, Loader2, Unlink } from "lucide-react";
import { useState } from "react";
import {
AlertDialog,
AlertDialogAction,
AlertDialogCancel,
AlertDialogContent,
AlertDialogDescription,
AlertDialogFooter,
AlertDialogHeader,
AlertDialogTitle,
AlertDialogTrigger,
} from "@/components/ui/alert-dialog";
import { Badge } from "@/components/ui/badge";
import { Button } from "@/components/ui/button";
import { Input } from "@/components/ui/input";
import { Label } from "@/components/ui/label";
import { PasswordInput } from "@/components/ui/password-input";
import { SettingsCardActions } from "@/components/settings/_card-actions";
import { SettingsCardHeader } from "@/components/settings/_card-header";
import { SettingsCard } from "@/components/settings/settings-card";
import { WrittenOutcomeLine } from "@/components/outcome/written-outcome-line";
import { SegmentedChoice } from "@/components/documents/sources/segmented-choice";
import {
errorCodeOf,
sourceErrorMessage,
} from "@/components/documents/sources/source-errors";
import {
systemName,
useDocumentSourcesStatus,
} from "@/components/documents/sources/use-document-sources";
import { apiDelete, apiPost, apiPut } from "@/lib/api/api-fetch";
import {
DOCUMENT_PICKER_SYSTEMS,
slugForSystem,
type DocumentPickerSystem,
type DocumentSourceConnectionDto,
} from "@/lib/documents/sources/types";
import { useFormatters, useTranslations } from "@/lib/i18n/context";
import { invalidateKeys, queryKeys } from "@/lib/query-keys";
type Outcome = { tone: "success" | "error"; message: string } | null;
export function DocumentSourcesCard() {
const { t } = useTranslations();
const { ownRecord, eligible, query } = useDocumentSourcesStatus();
const [picked, setPicked] = useState<DocumentPickerSystem>("PAPERLESS");
const data = query.data;
const connections = data?.connections ?? [];
// Editing needs the operator's list and the documents module. Without
// either, a connection that is still stored stays on the card so its token
// can be deleted; with none stored there is nothing to show.
const editable = eligible && data?.available === true;
if (!ownRecord || !data || (!editable && connections.length === 0)) {
return null;
}
const options: readonly DocumentPickerSystem[] = editable
? DOCUMENT_PICKER_SYSTEMS
: connections.map((c) => c.system);
const system = options.includes(picked) ? picked : options[0];
return (
<SettingsCard data-testid="document-sources-card">
<SettingsCardHeader
icon={Archive}
title={t("settings.documentSources.title")}
description={t("settings.documentSources.description")}
status={
connections.length > 0 ? (
<Badge variant="outline">
{t("settings.documentSources.statusConnected")}
</Badge>
) : null
}
/>
<p className="text-sm">
{editable
? t("settings.documentSources.explainer")
: eligible
? t("settings.documentSources.unavailableExplainer")
: t("settings.documentSources.moduleOffExplainer")}
</p>
{options.length > 1 ? (
<SegmentedChoice
options={options}
value={system}
onChange={setPicked}
label={t("settings.documentSources.systemLabel")}
slot="document-sources-system"
renderOption={(option) => (
<>
{systemName(option)}
{connections.some((c) => c.system === option) ? (
<Check
className="size-3.5"
aria-label={t("settings.documentSources.statusConnected")}
/>
) : null}
</>
)}
/>
) : null}
<SourceForm
// A fresh form per system: nothing typed for one leaks into the other.
key={system}
system={system}
connection={connections.find((c) => c.system === system) ?? null}
allowedOrigins={data.allowedOrigins}
editable={editable}
/>
</SettingsCard>
);
}
/** The origin of an address, or null while it is not one yet. */
function originOf(value: string): string | null {
try {
return new URL(value.trim()).origin;
} catch {
return null;
}
}
function SourceForm({
system,
connection,
allowedOrigins,
editable,
}: {
system: DocumentPickerSystem;
connection: DocumentSourceConnectionDto | null;
allowedOrigins: string[];
/** False when only removing the stored connection is possible. */
editable: boolean;
}) {
const { t } = useTranslations();
const format = useFormatters();
const queryClient = useQueryClient();
const slug = slugForSystem(system);
const isPapra = system === "PAPRA";
const [baseUrl, setBaseUrl] = useState(connection?.baseUrl ?? "");
const [organizationId, setOrganizationId] = useState(
connection?.organizationId ?? "",
);
const [token, setToken] = useState("");
const [outcome, setOutcome] = useState<Outcome>(null);
const dirty =
baseUrl.trim() !== (connection?.baseUrl ?? "") ||
(isPapra && organizationId.trim() !== (connection?.organizationId ?? "")) ||
token.trim() !== "";
const refresh = () =>
invalidateKeys(queryClient, [queryKeys.documentSources()]);
const save = useMutation({
mutationFn: () =>
apiPut<DocumentSourceConnectionDto>(`/api/documents/sources/${slug}`, {
baseUrl: baseUrl.trim(),
...(isPapra ? { organizationId: organizationId.trim() } : {}),
...(token.trim() ? { token: token.trim() } : {}),
}),
onMutate: () => setOutcome(null),
onSuccess: () => {
setToken("");
setOutcome({
tone: "success",
message: t("settings.documentSources.saved"),
});
void refresh();
},
onError: (err) =>
setOutcome({
tone: "error",
message: sourceErrorMessage(t, errorCodeOf(err)),
}),
});
const test = useMutation({
mutationFn: () =>
apiPost<{ ok: true; latencyMs: number }>(
`/api/documents/sources/${slug}/test`,
),
onMutate: () => setOutcome(null),
onSuccess: (data) => {
setOutcome({
tone: "success",
message: t("settings.documentSources.testOk", {
latency: data.latencyMs,
}),
});
void refresh();
},
onError: (err) =>
setOutcome({
tone: "error",
message: sourceErrorMessage(t, errorCodeOf(err)),
}),
});
const disconnect = useMutation({
mutationFn: () => apiDelete(`/api/documents/sources/${slug}`),
onSuccess: () => {
setBaseUrl("");
setOrganizationId("");
setToken("");
setOutcome(null);
void refresh();
},
onError: (err) =>
setOutcome({
tone: "error",
message: sourceErrorMessage(t, errorCodeOf(err)),
}),
});
const busy = save.isPending || test.isPending || disconnect.isPending;
// The stored token is reused only on the address's own origin; the server
// refuses to send it anywhere else, so the form asks for it again.
const keepsToken =
connection !== null &&
originOf(baseUrl) !== null &&
originOf(baseUrl) === originOf(connection.baseUrl);
const canSave =
editable &&
baseUrl.trim() !== "" &&
(!isPapra || organizationId.trim() !== "") &&
(keepsToken || token.trim() !== "") &&
dirty;
return (
<form
className="space-y-4"
onSubmit={(e) => {
e.preventDefault();
if (canSave && !busy) save.mutate();
}}
data-slot="document-sources-form"
>
<p
className="text-muted-foreground text-xs"
data-slot="document-sources-state"
>
{connection
? connection.lastVerifiedAt
? t("settings.documentSources.connectedSince", {
date: format.date(connection.lastVerifiedAt),
})
: t("settings.documentSources.statusConnected")
: t("settings.documentSources.statusNotConnected")}
</p>
{editable && connection && !connection.originAllowed ? (
<p
role="alert"
className="border-warning/30 bg-warning/10 text-foreground rounded-md border px-3 py-2 text-sm"
>
{t("settings.documentSources.originRemoved")}
</p>
) : null}
{editable ? (
<>
<div className="space-y-1.5">
<Label htmlFor={`document-source-url-${slug}`}>
{t("settings.documentSources.baseUrl")}
</Label>
<Input
id={`document-source-url-${slug}`}
type="url"
value={baseUrl}
onChange={(e) => setBaseUrl(e.target.value)}
placeholder={allowedOrigins[0] ?? "https://"}
maxLength={2048}
autoComplete="off"
inputMode="url"
spellCheck={false}
autoCapitalize="none"
/>
<p className="text-muted-foreground text-xs">
{t("settings.documentSources.allowedOrigins", {
origins: allowedOrigins.join(", "),
})}
</p>
</div>
{isPapra ? (
<div className="space-y-1.5">
<Label htmlFor="document-source-org">
{t("settings.documentSources.organizationId")}
</Label>
<Input
id="document-source-org"
value={organizationId}
onChange={(e) => setOrganizationId(e.target.value)}
maxLength={128}
autoComplete="off"
spellCheck={false}
autoCapitalize="none"
/>
<p className="text-muted-foreground text-xs">
{t("settings.documentSources.organizationHelp")}
</p>
</div>
) : null}
<div className="space-y-1.5">
<Label htmlFor={`document-source-token-${slug}`}>
{t("settings.documentSources.token")}
</Label>
<PasswordInput
id={`document-source-token-${slug}`}
value={token}
onChange={(e) => setToken(e.target.value)}
placeholder={
keepsToken
? t("settings.documentSources.tokenSaved")
: undefined
}
maxLength={512}
autoComplete="off"
spellCheck={false}
autoCapitalize="none"
/>
<p className="text-muted-foreground text-xs">
{isPapra
? t("settings.documentSources.tokenHelpPapra")
: t("settings.documentSources.tokenHelpPaperless")}
</p>
{connection && !keepsToken && baseUrl.trim() !== "" ? (
<p
className="text-muted-foreground text-xs"
data-slot="document-sources-token-again"
>
{t("settings.documentSources.tokenNewAddress")}
</p>
) : null}
</div>
</>
) : connection ? (
<p className="text-sm break-all" data-slot="document-sources-address">
{connection.baseUrl}
</p>
) : null}
{outcome ? (
<WrittenOutcomeLine
outcome={outcome.tone === "error" ? "failed" : "success"}
message={outcome.message}
testId="document-sources-outcome"
/>
) : null}
<SettingsCardActions>
{connection ? (
<AlertDialog>
<AlertDialogTrigger asChild>
<Button
type="button"
variant="destructive"
size="sm"
className="min-h-11"
disabled={busy}
>
<Unlink className="size-4" aria-hidden />
{t("settings.documentSources.disconnect")}
</Button>
</AlertDialogTrigger>
<AlertDialogContent>
<AlertDialogHeader>
<AlertDialogTitle>
{t("settings.documentSources.disconnectTitle", {
name: systemName(system),
})}
</AlertDialogTitle>
<AlertDialogDescription>
{t("settings.documentSources.disconnectDescription")}
</AlertDialogDescription>
</AlertDialogHeader>
<AlertDialogFooter>
<AlertDialogCancel>{t("common.cancel")}</AlertDialogCancel>
<AlertDialogAction onClick={() => disconnect.mutate()}>
{t("settings.documentSources.disconnect")}
</AlertDialogAction>
</AlertDialogFooter>
</AlertDialogContent>
</AlertDialog>
) : null}
{connection && editable ? (
<Button
type="button"
variant="outline"
size="sm"
className="min-h-11"
disabled={busy || dirty || !connection.originAllowed}
onClick={() => test.mutate()}
>
{test.isPending ? (
<Loader2
className="size-4 animate-spin motion-reduce:animate-none"
aria-hidden
/>
) : null}
{t("settings.documentSources.test")}
</Button>
) : null}
{editable ? (
<Button
type="submit"
size="sm"
className="min-h-11"
disabled={busy || !canSave}
>
{save.isPending ? (
<Loader2
className="size-4 animate-spin motion-reduce:animate-none"
aria-hidden
/>
) : null}
{save.isPending
? t("settings.documentSources.saving")
: t("settings.documentSources.save")}
</Button>
) : null}
</SettingsCardActions>
</form>
);
}
@@ -28,6 +28,8 @@ import { FolderOpen } from "lucide-react";
import { useTranslations } from "@/lib/i18n/context";
import { EntityLinkPicker } from "@/components/links/entity-link-picker";
import { useVaultDocumentOptions } from "@/components/links/vault-document-options";
import { ImportFromSourceButton } from "@/components/documents/sources/import-from-source-button";
import { mergeIds } from "@/components/documents/sources/merge-ids";
export function VaccinationDocumentPicker({
enabled,
@@ -73,6 +75,14 @@ export function VaccinationDocumentPicker({
options={vault.options}
searchPlaceholder={t("links.picker.searchPlaceholder")}
emptyLabel={t("vaccinations.form.linkNothingToOffer")}
extraAction={
// Joins the form's selection; the dose's save writes the links.
<ImportFromSourceButton
className="min-h-9"
kind="VACCINATION"
onImported={(ids) => onChange(mergeIds(documentIds, ids))}
/>
}
/>
</div>
);
+8
View File
@@ -161,6 +161,14 @@ export const ENCRYPTED_COLUMNS: readonly EncryptedColumn[] = [
{ model: "GoogleHealthConnection", field: "accessToken", kind: "string" },
{ model: "GoogleHealthConnection", field: "refreshToken", kind: "string" },
// ───── Document picker (v1.39.3, #1038) ─────
// A person's API token for their Paperless-ngx or Papra instance.
{
model: "DocumentSourceConnection",
field: "tokenEncrypted",
kind: "string",
},
// ───── AppSettings — operator credentials ─────
{ model: "AppSettings", field: "adminAiKeyEncrypted", kind: "string" },
{
+4
View File
@@ -155,6 +155,10 @@ export const WIPE_MODELS = [
// Further import keys that resolved to a document by content; a child of
// the document, removed before it so the counts are truthful.
"DocumentSourceAlias",
// The person's own connections to Paperless-ngx / Papra (#1038), with their
// encrypted API tokens. A wipe that left them would leave a working
// credential to another system behind an account that erased everything.
"DocumentSourceConnection",
"ExtractedFact",
"ImportJob",
+435
View File
@@ -0,0 +1,435 @@
/**
* One way into the vault for a file that has already been read into memory.
*
* The upload route (`POST /api/documents/inbound`, cookie, `["*"]` or a
* `documents:write` token) and the document picker's import route
* (`POST /api/documents/sources/{system}/import`, which fetches a picked
* document from Paperless-ngx or Papra) both store through here, so the rules a
* stored document is held to cannot differ by the way it arrived:
*
* - a source key answers before the bytes are looked at: a key already held
* (live, tombstoned, remembered as an alias or kept in the purge ledger) is
* a duplicate or a "deleted", never a second copy;
* - the per-file cap, the empty-file refusal and magic-byte classification
* (the wire Content-Type is never trusted);
* - pre-links to the caller's own live episodes and visits, refused when an
* id names nothing the caller owns;
* - sha256 duplicate detection against live rows, remembering the source key
* for the existing document so a later delete keeps that key deleted too;
* for an import, also against deleted (not yet purged) rows, which answer
* "deleted" rather than storing the file again;
* - the quota gate and the insert in one transaction under a per-user
* advisory lock, and the race on either partial unique index resolved the
* way the fast paths above would have answered;
* - the audit row, the wide-event annotation, and the background index,
* thumbnail and summary jobs (the summary only when the `documentAi`
* capability is open and the upload did not ask to defer AI reading).
*
* The callers own authentication, rate buckets, request parsing and the shape
* of their responses. This module returns what happened; it builds no HTTP.
*/
import type { Buffer } from "node:buffer";
import { createHash } from "node:crypto";
import type { AiCapabilityState } from "@/lib/ai/capabilities/types";
import { auditLog } from "@/lib/auth/audit";
import { prisma } from "@/lib/db";
import {
narrowOwnedEncounterIds,
narrowOwnedEpisodeIds,
} from "@/lib/documents/links";
import {
findSourceKey,
rememberSourceAlias,
type SourceKeyMatch,
} from "@/lib/documents/source-key";
import {
encryptDocumentContent,
type SerialisableDocument,
} from "@/lib/documents/store";
import { detectDocumentType } from "@/lib/documents/upload-policy";
import { enqueueDocumentIndex } from "@/lib/jobs/document-index";
import { enqueueDocumentSummary } from "@/lib/jobs/document-summary";
import { enqueueDocumentThumbnail } from "@/lib/jobs/document-thumbnail";
import { linkTargets } from "@/lib/links";
import { annotate } from "@/lib/logging/context";
import { isP2002 } from "@/lib/prisma-errors";
import type {
DocumentSourceSystemValue,
InboundDocumentKindValue,
} from "@/lib/validations/inbound-documents";
/**
* The person's own uploads per hour, from the web or the phone, and the
* picker's imports with them. A store-only upload touches no provider — the
* only abuse vector is disk, so a generous ceiling is enough. A narrow
* `documents:write` token draws on its own bucket instead
* (`resolveDocumentUploadLimitPerHour`).
*/
export const UPLOAD_LIMIT_PER_HOUR = 60;
export const UPLOAD_WINDOW_MS = 60 * 60 * 1000;
/** The bucket a person's own uploads and picker imports share. */
export function personalUploadBucket(userId: string): string {
return `documents-upload:${userId}`;
}
/** Parse a YYYY-MM-DD value into a UTC midnight Date. */
export function isoDateToUtc(value: string): Date {
return new Date(`${value}T00:00:00.000Z`);
}
export interface IngestInput {
userId: string;
/** A narrow `documents:write` token rather than a session or `["*"]`. */
scoped: boolean;
/** For the audit row. */
ipAddress: string | null;
bytes: Buffer;
filename: string | null;
title: string | null;
kind: InboundDocumentKindValue | null;
/** YYYY-MM-DD; the upload day when absent. */
documentDate: string | null;
episodeIds: string[];
encounterIds: string[];
sourceSystem: DocumentSourceSystemValue | null;
sourceId: string | null;
/**
* The source instance (origin) of the key, or null for a key sent without
* one; see `src/lib/documents/source-key.ts` for how null matches.
*/
sourceInstance: string | null;
/** Hold back automatic AI reading (`aiRead=defer`). */
aiDeferred: boolean;
/**
* The caller already asked about this source key (the upload's query-string
* form answers before the bucket is charged). Skips the second lookup.
*/
sourceKeyChecked: boolean;
/** The per-file cap and the quota, resolved by the caller. */
limits: { maxFileBytes: number; quotaBytes: number };
/**
* The `documentAi` capability, asked by the caller's route only once a new
* document was stored (a duplicate never asks). Passed in rather than asked
* here so each route that can queue AI work names the capability in its own
* source, where the AI route inventory reads it.
*/
documentAi: () => Promise<AiCapabilityState>;
}
export type IngestResult =
| { kind: "sourceKey"; match: SourceKeyMatch }
| { kind: "tooLarge"; maxFileBytes: number }
| { kind: "empty" }
| { kind: "unsupportedType" }
| { kind: "episodeNotFound" }
| { kind: "encounterNotFound" }
| { kind: "aliasLimit" }
| { kind: "duplicate"; document: SerialisableDocument }
| { kind: "quotaExceeded"; usedBytes: number; quotaBytes: number }
| {
kind: "stored";
document: SerialisableDocument;
servingClass: string;
linkedEpisodes: number;
linkedVisits: number;
};
/** Internal signal: the quota gate inside the insert transaction tripped. */
class QuotaExceededError extends Error {
constructor(public readonly usedBytes: number) {
super("Document quota exceeded");
this.name = "QuotaExceededError";
}
}
/** A live row with the same bytes, remembering the source key for it. */
async function answerDuplicate(
input: IngestInput,
existing: SerialisableDocument,
): Promise<IngestResult> {
// An import sending bytes that are already stored under another key (or
// none) is answered with that document — and the key is remembered for it,
// so once the person deletes the document this key stays deleted too.
if (input.sourceSystem && input.sourceId) {
const remembered = await rememberSourceAlias(
input.userId,
existing.id,
input.sourceSystem,
input.sourceId,
input.sourceInstance,
);
if (remembered === "limit") return { kind: "aliasLimit" };
}
return { kind: "duplicate", document: existing };
}
export async function ingestDocument(
input: IngestInput,
): Promise<IngestResult> {
const { userId, sourceSystem, sourceId, limits, bytes } = input;
const sourceInstance = sourceId ? input.sourceInstance : null;
// A source key answers before the bytes are looked at: an import re-sending
// what it sent before is a duplicate, and one re-sending a document the
// person deleted is refused a second copy — live and tombstoned rows both
// count (the unique index has no `deleted_at` predicate), and past the purge
// the ledger remembers.
if (sourceSystem && sourceId && !input.sourceKeyChecked) {
const match = await findSourceKey(
userId,
sourceSystem,
sourceId,
sourceInstance,
);
if (match) return { kind: "sourceKey", match };
}
if (bytes.byteLength > limits.maxFileBytes) {
return { kind: "tooLarge", maxFileBytes: limits.maxFileBytes };
}
if (bytes.byteLength === 0) return { kind: "empty" };
// Magic-byte classification; the wire Content-Type is never trusted.
const detected = detectDocumentType(bytes, input.filename);
if (!detected) return { kind: "unsupportedType" };
// Pre-linking: every episode id must be a LIVE episode of the caller.
const episodeIds = await narrowOwnedEpisodeIds(userId, input.episodeIds);
if (episodeIds === null) return { kind: "episodeNotFound" };
// Same for the visit ids the review step offered. A refusal rather than a
// silent drop: the person saw a visit named and would otherwise watch the
// upload succeed without the link they asked for.
const encounterIds = await narrowOwnedEncounterIds(
userId,
input.encounterIds,
);
if (encounterIds === null) return { kind: "encounterNotFound" };
// sha256 of the PLAINTEXT for same-user duplicate detection.
const contentSha256 = createHash("sha256").update(bytes).digest("hex");
// Fast-path dedupe check; the partial unique index closes the race below.
const existing = await prisma.inboundDocument.findFirst({
where: { userId, contentSha256, deletedAt: null },
omit: { contentEncrypted: true },
});
if (existing) return answerDuplicate(input, existing);
// An import (a source key is present) of bytes the person deleted here,
// arriving under a key HealthLog has not seen: another system, another
// instance, or a re-scan with a new id. The deletion was a decision about
// the document, not about one key, so it is answered "deleted", and the
// new key is remembered on the tombstone so the purge carries it into the
// ledger. After the 30-day purge the bytes and their hash are gone and only
// the keys remain; no content hash of a deleted document is kept. A person
// uploading the file by hand (no key) is making a new copy on purpose.
if (sourceSystem && sourceId) {
const tombstone = await prisma.inboundDocument.findFirst({
where: { userId, contentSha256, deletedAt: { not: null } },
select: { id: true },
orderBy: { deletedAt: "desc" },
});
if (tombstone) {
const remembered = await rememberSourceAlias(
userId,
tombstone.id,
sourceSystem,
sourceId,
sourceInstance,
);
if (remembered === "limit") return { kind: "aliasLimit" };
return {
kind: "sourceKey",
match: { state: "deleted", id: tombstone.id },
};
}
}
// Quota gate + insert + pre-links in ONE transaction. Usage counts every
// non-purged row — tombstones still hold TOAST bytes, so "deleted" bytes
// are never invisible weight (undo-delete never changes usage).
let document: SerialisableDocument;
try {
document = await prisma.$transaction(async (tx) => {
// Serialise the quota gate per user: without this, N concurrent
// uploads all read the same SUM before any of them commits and the
// quota can be overshot by up to N × cap in one burst. The advisory
// lock is transaction-scoped (released on commit/rollback) and keyed
// on the user id, so uploads by different users never queue on each
// other.
// (`pg_advisory_xact_lock` returns void, which the client cannot
// deserialize as a column — selecting FROM it yields a plain int row.)
await tx.$queryRaw`
SELECT 1 AS locked
FROM pg_advisory_xact_lock(hashtextextended('documents-quota:' || ${userId}, 0))
`;
const rows = await tx.$queryRaw<Array<{ used: bigint }>>`
SELECT COALESCE(SUM(byte_size), 0)::bigint AS used
FROM inbound_documents
WHERE user_id = ${userId}
`;
const usedBytes = Number(rows[0]?.used ?? 0);
if (usedBytes + bytes.byteLength > limits.quotaBytes) {
throw new QuotaExceededError(usedBytes);
}
const { content, codec } = encryptDocumentContent(bytes);
// No mass assignment — every column is set field-by-field; `userId`
// comes from the session, never the body. `documentDate` defaults to
// the upload day so display == sort == filter (user-editable later).
const created = await tx.inboundDocument.create({
data: {
userId,
kind: input.kind ?? "OTHER",
// Stored plaintext on purpose (mirrors `filename`) so the list can
// ILIKE-search + ORDER BY it. It MAY hold PHI the user types; that
// is the accepted tradeoff for server-side search/sort — the
// document body stays encrypted.
title: input.title,
filename: input.filename ? input.filename.slice(0, 255) : null,
mimeType: detected.mimeType,
byteSize: bytes.byteLength,
contentEncrypted: content,
contentCodec: codec,
contentSha256,
status: "STORED",
documentDate: input.documentDate
? isoDateToUtc(input.documentDate)
: new Date(),
sourceSystem,
sourceId,
sourceInstance,
aiReadDeferred: input.aiDeferred,
},
omit: { contentEncrypted: true },
});
if (episodeIds.length > 0) {
await linkTargets(tx, {
userId,
sourceKind: "document",
sourceId: created.id,
targetKind: "conditionEpisode",
targetIds: episodeIds,
});
}
if (encounterIds.length > 0) {
await linkTargets(tx, {
userId,
sourceKind: "document",
sourceId: created.id,
targetKind: "encounter",
targetIds: encounterIds,
});
}
return created;
});
} catch (err) {
if (err instanceof QuotaExceededError) {
return {
kind: "quotaExceeded",
usedBytes: err.usedBytes,
quotaBytes: limits.quotaBytes,
};
}
if (isP2002(err)) {
// A racing upload won one of the two partial unique indexes — the same
// source key, or the same bytes. Surface the winner exactly as the fast
// paths above would have.
if (sourceSystem && sourceId) {
const match = await findSourceKey(
userId,
sourceSystem,
sourceId,
sourceInstance,
);
if (match) return { kind: "sourceKey", match };
}
const winner = await prisma.inboundDocument.findFirst({
where: { userId, contentSha256, deletedAt: null },
omit: { contentEncrypted: true },
});
if (winner) return answerDuplicate(input, winner);
}
throw err;
}
await auditLog("documents.inbound.store", {
userId,
ipAddress: input.ipAddress,
details: {
documentId: document.id,
mime: detected.mimeType,
...(input.scoped ? { scoped: true } : {}),
...(sourceSystem ? { sourceSystem } : {}),
},
});
annotate({
action: { name: "documents.vault.upload" },
meta: {
documentId: document.id,
byteSize: document.byteSize,
servingClass: detected.servingClass,
linked: episodeIds.length,
linkedVisits: encounterIds.length,
scoped: input.scoped,
sourceSystem,
aiDeferred: input.aiDeferred,
},
});
// Auto-index the freshly stored document for content search: enqueue a
// fire-and-forget background job (provider-first, local text-layer fallback).
// The caller never blocks on or fails because of indexing — the enqueue is
// not awaited and swallows its own errors (a missing boss or a transient
// send failure is a silent no-op). Only fresh inserts enqueue — a duplicate
// returns early above and never reaches here.
//
// `aiRead=defer` narrows it to the local text layer: an import of a whole
// archive must not turn into one provider call per file. The person reads
// them later, deliberately, from the document itself.
if (input.aiDeferred) {
void enqueueDocumentIndex(userId, document.id, { localOnly: true });
} else {
void enqueueDocumentIndex(userId, document.id);
}
// Render a preview thumbnail in the background too (pure local compute — no
// egress). Same fire-and-forget contract; a dropped enqueue is recoverable
// via the boot backfill.
void enqueueDocumentThumbnail(userId, document.id);
// Summarise the freshly stored document in the background — but ONLY when the
// `documentsAutoAiRead` opt-in is ON (the job re-checks it and the egress
// consent, and no-ops otherwise) and the `documentAi` capability is open for
// this record. Storing never depends on AI: the document is stored above
// whatever the capability says, and the index job still runs its
// provider-free text-layer path.
//
// A deferred upload skips it outright and keeps `summaryState: NONE`, so
// the detail sheet offers "Generate summary" rather than a pending state.
const documentAi = input.aiDeferred
? { available: false as const, reason: "deferred" }
: await input.documentAi();
if (documentAi.available) {
void enqueueDocumentSummary(userId, document.id);
} else {
annotate({
action: { name: "documents.summary.enqueueSkipped" },
meta: { documentId: document.id, reason: documentAi.reason },
});
}
return {
kind: "stored",
document,
servingClass: detected.servingClass,
linkedEpisodes: episodeIds.length,
linkedVisits: encounterIds.length,
};
}
+130 -12
View File
@@ -11,8 +11,21 @@
* 3. `DocumentImportKey` — a key whose document the purge has removed.
*
* The first two resolve to a document (live → duplicate, tombstoned →
* deleted); the third only to "deleted". The upload route and the lookup
* route both ask here, so they cannot disagree about what a key means.
* deleted); the third only to "deleted". The upload route, the lookup route
* and the document picker all ask here, so they cannot disagree about what a
* key means.
*
* v1.39.3 — a key also names its instance (`sourceInstance`, the origin of
* the Paperless-ngx or Papra it came from): id 123 on one Paperless and id
* 123 on another are different documents. A null on either side is a
* wildcard, so nothing stored before the column existed is forgotten:
*
* - a key stored without an instance (a v1.39.2 import, an older script, a
* workflow that does not send one) matches a lookup for any instance of
* its system;
* - a lookup without an instance matches a key stored for any instance.
*
* Where both an exact and a wildcard match exist, the exact one wins.
*/
import { prisma } from "@/lib/db";
import { checkRateLimit, type RateLimitResult } from "@/lib/rate-limit";
@@ -23,22 +36,44 @@ export type SourceKeyMatch =
| { state: "live"; document: SerialisableDocument }
| { state: "deleted"; id: string | null };
/**
* The instance half of a key match: the same instance or a stored key without
* one; any instance when the lookup has none.
*/
export function instanceMatch(sourceInstance: string | null): {
OR?: Array<{ sourceInstance: string | null }>;
} {
return sourceInstance === null
? {}
: { OR: [{ sourceInstance }, { sourceInstance: null }] };
}
/** Exact instance first, then a key stored without one (nulls sort last). */
const EXACT_FIRST = { sourceInstance: { sort: "asc", nulls: "last" } } as const;
export async function findSourceKey(
userId: string,
sourceSystem: DocumentSourceSystemValue,
sourceId: string,
sourceInstance: string | null = null,
): Promise<SourceKeyMatch | null> {
const key = {
userId,
sourceSystem,
sourceId,
...instanceMatch(sourceInstance),
};
const own = await prisma.inboundDocument.findFirst({
where: { userId, sourceSystem, sourceId },
where: key,
omit: { contentEncrypted: true },
orderBy: EXACT_FIRST,
});
const viaAlias = own
? null
: await prisma.documentSourceAlias.findUnique({
where: {
userId_sourceSystem_sourceId: { userId, sourceSystem, sourceId },
},
: await prisma.documentSourceAlias.findFirst({
where: key,
select: { document: { omit: { contentEncrypted: true } } },
orderBy: EXACT_FIRST,
});
const document = own ?? viaAlias?.document ?? null;
if (document) {
@@ -46,10 +81,8 @@ export async function findSourceKey(
? { state: "deleted", id: document.id }
: { state: "live", document };
}
const purged = await prisma.documentImportKey.findUnique({
where: {
userId_sourceSystem_sourceId: { userId, sourceSystem, sourceId },
},
const purged = await prisma.documentImportKey.findFirst({
where: key,
select: { id: true },
});
return purged ? { state: "deleted", id: null } : null;
@@ -74,13 +107,14 @@ export async function rememberSourceAlias(
documentId: string,
sourceSystem: DocumentSourceSystemValue,
sourceId: string,
sourceInstance: string | null = null,
): Promise<"remembered" | "known" | "limit"> {
const held = await prisma.documentSourceAlias.count({
where: { userId, documentId },
});
if (held >= MAX_SOURCE_ALIASES_PER_DOCUMENT) return "limit";
const { count } = await prisma.documentSourceAlias.createMany({
data: [{ userId, documentId, sourceSystem, sourceId }],
data: [{ userId, documentId, sourceSystem, sourceInstance, sourceId }],
skipDuplicates: true,
});
return count > 0 ? "remembered" : "known";
@@ -107,3 +141,87 @@ export function checkSourceLookupRateLimit(
SOURCE_LOOKUP_WINDOW_MS,
);
}
/** What the vault holds for one source key, as the picker lists it. */
export type SourceKeyState =
{ state: "imported"; documentId: string } | { state: "deleted" };
/**
* {@link findSourceKey} for a page of results at once: the same three places,
* three queries for the whole page instead of three per result. A key held by
* a live document answers `imported` with its id; one held by a tombstone, or
* kept by the purge ledger, answers `deleted`. A key the vault does not know
* is absent from the map.
*/
export async function findSourceKeyStates(
userId: string,
sourceSystem: DocumentSourceSystemValue,
sourceIds: string[],
sourceInstance: string | null = null,
): Promise<Map<string, SourceKeyState>> {
const states = new Map<string, SourceKeyState>();
const ids = [...new Set(sourceIds)];
if (ids.length === 0) return states;
const where = {
userId,
sourceSystem,
sourceId: { in: ids },
...instanceMatch(sourceInstance),
};
const [own, aliases, purged] = await Promise.all([
prisma.inboundDocument.findMany({
where,
select: {
id: true,
sourceId: true,
sourceInstance: true,
deletedAt: true,
},
}),
prisma.documentSourceAlias.findMany({
where,
select: {
sourceId: true,
sourceInstance: true,
document: { select: { id: true, deletedAt: true } },
},
}),
prisma.documentImportKey.findMany({
where,
select: { sourceId: true },
}),
]);
// A wildcard (instance-less) row first, so an exact-instance row after it
// wins, as it does in `findSourceKey`.
const wildcardFirst = <T extends { sourceInstance: string | null }>(
rows: T[],
) =>
[...rows].sort(
(a, b) =>
Number(a.sourceInstance === sourceInstance) -
Number(b.sourceInstance === sourceInstance),
);
// Lowest precedence first, so the document's own key wins, as it does in
// `findSourceKey`.
for (const row of purged) states.set(row.sourceId, { state: "deleted" });
for (const row of wildcardFirst(aliases)) {
states.set(
row.sourceId,
row.document.deletedAt
? { state: "deleted" }
: { state: "imported", documentId: row.document.id },
);
}
for (const row of wildcardFirst(own)) {
if (!row.sourceId) continue;
states.set(
row.sourceId,
row.deletedAt
? { state: "deleted" }
: { state: "imported", documentId: row.id },
);
}
return states;
}
@@ -0,0 +1,642 @@
/**
* The Paperless-ngx and Papra adapters against recorded response shapes, with
* the network mocked at `safeFetch`. What these pin:
*
* - every request carries the listed origin as the operator-approved pin and
* forbids redirects, and a 3xx is reported, not followed;
* - the source's own query parameters (name, tag, date, page), and the local
* re-check that keeps a server ignoring a filter from widening a result;
* - the error mapping (401 → authRefused, 406 / an old X-Api-Version →
* versionTooOld, 404 → notFound, non-JSON → badResponse, a refused or dead
* host → unreachable);
* - the byte caps on JSON and on a download.
*/
import { beforeEach, describe, expect, it, vi } from "vitest";
const safeFetch = vi.hoisted(() => vi.fn());
vi.mock("@/lib/safe-fetch", async (importOriginal) => {
const actual = await importOriginal<typeof import("@/lib/safe-fetch")>();
return { ...actual, safeFetch };
});
import { SafeFetchError } from "@/lib/safe-fetch";
import { sourceClient } from "../clients";
import {
DocumentSourceError,
DocumentSourceTooLargeError,
SOURCE_JSON_MAX_BYTES,
} from "../http";
const PAPERLESS = {
system: "PAPERLESS" as const,
origin: "http://paperless.lan:8000",
baseUrl: "http://paperless.lan:8000/paperless",
organizationId: null,
token: "pl-secret-token",
};
const PAPRA = {
system: "PAPRA" as const,
origin: "https://papra.example.com",
baseUrl: "https://papra.example.com",
organizationId: "org_a1b2c3d4e5f6g7h8i9j0k1l2",
token: "ppk_secret",
};
function json(body: unknown, init: ResponseInit = {}): Response {
return new Response(JSON.stringify(body), {
status: 200,
headers: { "content-type": "application/json" },
...init,
});
}
/** Route mocked requests by path; anything unmatched is a 404. */
function serve(routes: Record<string, (url: URL) => Response>) {
safeFetch.mockImplementation(async (target: string) => {
const url = new URL(target);
for (const [prefix, handler] of Object.entries(routes)) {
if (url.pathname.startsWith(prefix)) return handler(url);
}
return new Response("not found", { status: 404 });
});
}
function calledUrls(): URL[] {
return safeFetch.mock.calls.map((c) => new URL(String(c[0])));
}
async function failure(promise: Promise<unknown>) {
try {
await promise;
} catch (err) {
return err;
}
throw new Error("expected a failure");
}
beforeEach(() => {
safeFetch.mockReset();
});
describe("every request is pinned to the listed origin", () => {
it("passes the origin as the operator-approved pin and forbids redirects", async () => {
serve({ "/paperless/api/documents/": () => json({ results: [] }) });
await sourceClient(PAPERLESS).test();
expect(safeFetch).toHaveBeenCalledTimes(1);
const [target, init, opts] = safeFetch.mock.calls[0];
expect(String(target)).toBe(
"http://paperless.lan:8000/paperless/api/documents/?page_size=1&fields=id",
);
expect(init.redirect).toBe("manual");
expect(init.headers.Authorization).toBe("Token pl-secret-token");
expect(init.headers.Accept).toBe("application/json; version=9");
expect(opts.operatorApprovedPrivateOrigin).toBe(
"http://paperless.lan:8000",
);
expect(opts.timeoutMs).toBeGreaterThan(0);
expect(opts.requirePublicHost).toBeUndefined();
});
it("reports a redirect instead of following it", async () => {
serve({
"/paperless/api/documents/": () =>
new Response(null, {
status: 302,
headers: { location: "http://169.254.169.254/latest/meta-data" },
}),
});
const err = await failure(sourceClient(PAPERLESS).test());
expect(err).toBeInstanceOf(DocumentSourceError);
expect((err as DocumentSourceError).code).toBe("redirected");
expect(safeFetch).toHaveBeenCalledTimes(1);
});
it("reports a refused or dead destination as unreachable", async () => {
safeFetch.mockRejectedValue(new SafeFetchError("refused", "private_host"));
const err = await failure(sourceClient(PAPRA).test());
expect((err as DocumentSourceError).code).toBe("unreachable");
});
});
describe("Paperless-ngx", () => {
const tags = {
results: [
{ id: 7, name: "Health" },
{ id: 9, name: "Insurance" },
],
next: null,
};
it("searches by title with tag, date and page, and names the tags", async () => {
serve({
"/paperless/api/tags/": () => json(tags),
"/paperless/api/documents/": () =>
json({
count: 30,
next: "http://paperless.lan:8000/paperless/api/documents/?page=3",
results: [
{
id: 41,
title: "Blood test",
created: "2025-03-04",
tags: [7],
original_file_name: "labor.pdf",
},
// A server that ignored the tag filter must not widen the result.
{ id: 42, title: "Car", created: "2025-03-05", tags: [9] },
// Nor one that ignored the date filter.
{ id: 43, title: "Old", created: "2019-01-01", tags: [7] },
// No title: the file name without its extension.
{
id: 44,
title: " ",
created: "2025-06-01T10:00:00+02:00",
tags: [7],
original_file_name: "discharge-letter.pdf",
},
],
}),
});
const result = await sourceClient(PAPERLESS).search({
q: "blood",
tagId: "7",
from: "2025-01-01",
to: "2025-12-31",
page: 2,
});
const list = calledUrls().find((u) => u.pathname.endsWith("/documents/"));
expect(list?.searchParams.get("title__icontains")).toBe("blood");
expect(list?.searchParams.get("tags__id__all")).toBe("7");
expect(list?.searchParams.get("created__gte")).toBe("2025-01-01");
expect(list?.searchParams.get("created__lte")).toBe("2025-12-31");
// API version 9 made `created` a date; the `__date` lookups are gone.
expect(list?.searchParams.has("created__date__gte")).toBe(false);
expect(list?.searchParams.get("page")).toBe("2");
expect(list?.searchParams.get("page_size")).toBe("25");
expect(result.hasMore).toBe(true);
expect(result.items).toEqual([
{
sourceId: "41",
title: "Blood test",
date: "2025-03-04",
tags: ["Health"],
sizeBytes: null,
},
{
sourceId: "44",
title: "discharge-letter",
date: "2025-06-01",
tags: ["Health"],
sizeBytes: null,
},
]);
});
it("answers a page past the end with nothing", async () => {
serve({
"/paperless/api/tags/": () => json(tags),
"/paperless/api/documents/": () =>
json({ detail: "Invalid page." }, { status: 404 }),
});
await expect(
sourceClient(PAPERLESS).search({
q: "",
tagId: null,
from: null,
to: null,
page: 99,
}),
).resolves.toEqual({ items: [], hasMore: false });
});
it("maps a refused token, an unsupported version and an old server", async () => {
serve({ "/paperless/api/": () => json({}, { status: 401 }) });
expect(
((await failure(sourceClient(PAPERLESS).test())) as DocumentSourceError)
.code,
).toBe("authRefused");
serve({ "/paperless/api/": () => json({}, { status: 406 }) });
expect(
((await failure(sourceClient(PAPERLESS).test())) as DocumentSourceError)
.code,
).toBe("versionTooOld");
serve({
"/paperless/api/": () =>
json({ results: [] }, { headers: { "x-api-version": "5" } }),
});
expect(
((await failure(sourceClient(PAPERLESS).test())) as DocumentSourceError)
.code,
).toBe("versionTooOld");
});
it("reports a web page where the API should be as a bad response", async () => {
serve({
"/paperless/api/": () =>
new Response("<html>login</html>", {
headers: { "content-type": "text/html" },
}),
});
const err = await failure(sourceClient(PAPERLESS).test());
expect((err as DocumentSourceError).code).toBe("badResponse");
});
it("refuses a JSON answer larger than the cap", async () => {
serve({
"/paperless/api/": () =>
new Response("x".repeat(SOURCE_JSON_MAX_BYTES + 10), { status: 200 }),
});
const err = await failure(sourceClient(PAPERLESS).test());
expect((err as DocumentSourceError).code).toBe("badResponse");
});
it("reads a document's details and downloads the original, within the cap", async () => {
serve({
"/paperless/api/documents/41/download/": () =>
new Response(new Uint8Array([37, 80, 68, 70]), {
headers: { "content-disposition": 'attachment; filename="x.pdf"' },
}),
"/paperless/api/documents/41/": () =>
json({
id: 41,
title: "Blood test",
created: "2025-03-04",
original_file_name: "labor.pdf",
}),
});
const client = sourceClient(PAPERLESS);
await expect(client.document("41")).resolves.toEqual({
title: "Blood test",
date: "2025-03-04",
filename: "labor.pdf",
});
const file = await client.download("41", 1024);
expect([...file.bytes]).toEqual([37, 80, 68, 70]);
expect(file.filename).toBe("x.pdf");
const download = calledUrls().at(-1);
expect(download?.searchParams.get("original")).toBe("true");
});
it("refuses a download larger than the vault accepts, declared or not", async () => {
serve({
"/paperless/api/documents/41/download/": () =>
new Response(new Uint8Array(2048), {
headers: { "content-length": "2048" },
}),
});
expect(
await failure(sourceClient(PAPERLESS).download("41", 1024)),
).toBeInstanceOf(DocumentSourceTooLargeError);
serve({
"/paperless/api/documents/41/download/": () =>
new Response(
new ReadableStream({
start(controller) {
controller.enqueue(new Uint8Array(800));
controller.enqueue(new Uint8Array(800));
controller.close();
},
}),
),
});
expect(
await failure(sourceClient(PAPERLESS).download("41", 1024)),
).toBeInstanceOf(DocumentSourceTooLargeError);
});
});
describe("ids never shape a request path", () => {
it("refuses a Paperless id that is not a number, and dot segments, before dialling", async () => {
const client = sourceClient(PAPERLESS);
for (const id of ["..", ".", "41/../x", "abc", "41?x=1"]) {
const err = await failure(client.document(id));
expect((err as DocumentSourceError).code, id).toBe("notFound");
expect(
((await failure(client.download(id, 10))) as DocumentSourceError).code,
).toBe("notFound");
}
expect(safeFetch).not.toHaveBeenCalled();
});
it("refuses a Papra id or organization id outside [A-Za-z0-9_-]", async () => {
for (const id of ["..", "doc/1", "doc%2F1", "doc 1"]) {
await failure(sourceClient(PAPRA).document(id));
}
expect(() =>
sourceClient({ ...PAPRA, organizationId: "../admin" }),
).toThrow(DocumentSourceError);
expect(safeFetch).not.toHaveBeenCalled();
});
it("leaves a result with an unusable id out of the list", async () => {
serve({
"/paperless/api/tags/": () => json({ results: [], next: null }),
"/paperless/api/documents/": () =>
json({
next: null,
results: [
{ id: "..", title: "Odd", created: "2025-01-01", tags: [] },
{ id: 5, title: "Fine", created: "2025-01-01", tags: [] },
],
}),
});
const result = await sourceClient(PAPERLESS).search({
q: "",
tagId: null,
from: null,
to: null,
page: 1,
});
expect(result.items.map((i) => i.sourceId)).toEqual(["5"]);
});
});
describe("Paperless-ngx tags", () => {
it("pages through tags beyond the first 250", async () => {
serve({
"/paperless/api/tags/": (url) => {
const page = Number(url.searchParams.get("page"));
const results = Array.from(
{ length: page === 1 ? 250 : 3 },
(_, i) => ({
id: page * 1000 + i,
name: `t${page}-${i}`,
}),
);
return json({ results, next: page === 1 ? "more" : null });
},
});
const tags = await sourceClient(PAPERLESS).tags();
expect(tags).toHaveLength(253);
expect(calledUrls().map((u) => u.searchParams.get("page"))).toEqual([
"1",
"2",
]);
});
});
describe("Papra, as it answers for real", () => {
it("quotes each word so a leading dash is not a negation, and sends the date range", async () => {
serve({
"/api/organizations/org_a1b2c3d4e5f6g7h8i9j0k1l2/documents": () =>
json({ documents: [], documentsCount: 0 }),
});
await sourceClient(PAPRA).search({
q: "-Knie Entlass",
tagId: null,
from: "2024-01-01",
to: "2024-12-31",
page: 1,
});
const list = calledUrls().find((u) => u.pathname.endsWith("/documents"));
expect(list?.searchParams.get("searchQuery")).toBe(
'"-Knie" "Entlass" date:>=2024-01-01 date:<=2024-12-31',
);
});
it("dates a search row by its document date only, as Papra's date filter does", async () => {
serve({
"/api/organizations/org_a1b2c3d4e5f6g7h8i9j0k1l2/documents": () =>
json({
documentsCount: 2,
documents: [
{
id: "doc_dated",
name: "A.pdf",
documentDate: "2024-05-01",
createdAt: "2025-01-01T00:00:00.000Z",
},
{
id: "doc_undated",
name: "B.pdf",
createdAt: "2025-01-01T00:00:00.000Z",
},
],
}),
});
const all = await sourceClient(PAPRA).search({
q: "",
tagId: null,
from: null,
to: null,
page: 1,
});
expect(all.items.map((i) => [i.sourceId, i.date])).toEqual([
["doc_dated", "2024-05-01"],
["doc_undated", null],
]);
const ranged = await sourceClient(PAPRA).search({
q: "",
tagId: null,
from: "2024-01-01",
to: "2025-12-31",
page: 1,
});
// Papra's date: filter never matches a document without a document date;
// the per-row check agrees instead of re-admitting it by upload day.
expect(ranged.items.map((i) => i.sourceId)).toEqual(["doc_dated"]);
});
it("still files an undated Papra document under the day it was added on import", async () => {
serve({
"/api/organizations/org_a1b2c3d4e5f6g7h8i9j0k1l2/documents/doc_undated":
() =>
json({
document: {
id: "doc_undated",
name: "B.pdf",
createdAt: "2025-01-01T09:00:00.000Z",
},
}),
});
await expect(
sourceClient(PAPRA).document("doc_undated"),
).resolves.toMatchObject({
date: "2025-01-01",
});
});
it("refuses an organization id Papra would not issue", () => {
for (const org of ["org_123", "../admin", "ORG_A1B2C3D4E5F6G7H8I9J0K1L2"]) {
expect(() => sourceClient({ ...PAPRA, organizationId: org })).toThrow(
DocumentSourceError,
);
}
expect(safeFetch).not.toHaveBeenCalled();
});
it("reads 'not in this organization' and a malformed id as no such organization", async () => {
serve({
"/api/organizations/": () =>
json({ error: { code: "user.not_in_organization" } }, { status: 403 }),
});
expect(
((await failure(sourceClient(PAPRA).test())) as DocumentSourceError).code,
).toBe("notFound");
serve({
"/api/organizations/": () =>
json(
{ error: { code: "server.invalid_request.params" } },
{ status: 400 },
),
});
expect(
((await failure(sourceClient(PAPRA).test())) as DocumentSourceError).code,
).toBe("notFound");
// Any other refusal of the key is still the key.
serve({
"/api/organizations/": () =>
json({ error: { code: "auth.unauthorized" } }, { status: 401 }),
});
expect(
((await failure(sourceClient(PAPRA).test())) as DocumentSourceError).code,
).toBe("authRefused");
});
it("fails the connection test when the key cannot read tags", async () => {
serve({
"/api/organizations/org_a1b2c3d4e5f6g7h8i9j0k1l2/tags": () =>
json({}, { status: 401 }),
"/api/organizations/org_a1b2c3d4e5f6g7h8i9j0k1l2/documents": () =>
json({ documents: [], documentsCount: 0 }),
});
const err = await failure(sourceClient(PAPRA).test());
expect((err as DocumentSourceError).code).toBe("permissionMissing");
});
});
describe("Papra", () => {
const tags = {
tags: [
{ id: "tag_b", name: "Lab results" },
{ id: "tag_a", name: "Health" },
],
};
it("sends the name and the tag through searchQuery and re-checks both", async () => {
serve({
"/api/organizations/org_a1b2c3d4e5f6g7h8i9j0k1l2/tags": () => json(tags),
"/api/organizations/org_a1b2c3d4e5f6g7h8i9j0k1l2/documents": () =>
json({
documentsCount: 60,
documents: [
{
id: "doc_1",
name: "Blood test.pdf",
originalName: "Blood test.pdf",
originalSize: 1234,
documentDate: "2025-03-04",
createdAt: "2025-03-04T09:00:00.000Z",
tags: [{ id: "tag_b", name: "Lab results" }],
},
{
id: "doc_2",
name: "Other.pdf",
createdAt: "2025-03-04T09:00:00.000Z",
tags: [{ id: "tag_a", name: "Health" }],
},
{
id: "doc_3",
name: "Too old.pdf",
documentDate: "2018-01-01",
createdAt: "2025-03-04T09:00:00.000Z",
tags: [{ id: "tag_b", name: "Lab results" }],
},
],
}),
});
const result = await sourceClient(PAPRA).search({
q: "blood",
tagId: "tag_b",
from: "2025-01-01",
to: null,
page: 1,
});
const list = calledUrls().find((u) => u.pathname.endsWith("/documents"));
expect(list?.searchParams.get("searchQuery")).toBe(
'"blood" tag:"Lab results" date:>=2025-01-01',
);
expect(list?.searchParams.get("pageIndex")).toBe("0");
expect(list?.searchParams.get("pageSize")).toBe("25");
expect(safeFetch.mock.calls[0][1].headers.Authorization).toBe(
"Bearer ppk_secret",
);
expect(result).toEqual({
items: [
{
sourceId: "doc_1",
title: "Blood test",
date: "2025-03-04",
tags: ["Lab results"],
sizeBytes: 1234,
},
],
hasMore: true,
});
});
it("matches nothing for a tag that no longer exists", async () => {
serve({
"/api/organizations/org_a1b2c3d4e5f6g7h8i9j0k1l2/tags": () => json(tags),
});
await expect(
sourceClient(PAPRA).search({
q: "",
tagId: "tag_gone",
from: null,
to: null,
page: 1,
}),
).resolves.toEqual({ items: [], hasMore: false });
expect(calledUrls().some((u) => u.pathname.endsWith("/documents"))).toBe(
false,
);
});
it("lists tags by name and maps an unknown organization to notFound", async () => {
serve({
"/api/organizations/org_a1b2c3d4e5f6g7h8i9j0k1l2/tags": () => json(tags),
});
await expect(sourceClient(PAPRA).tags()).resolves.toEqual([
{ id: "tag_a", name: "Health" },
{ id: "tag_b", name: "Lab results" },
]);
serve({});
const err = await failure(sourceClient(PAPRA).test());
expect((err as DocumentSourceError).code).toBe("notFound");
});
it("reads a document and downloads its file", async () => {
serve({
"/api/organizations/org_a1b2c3d4e5f6g7h8i9j0k1l2/documents/doc_1/file":
() => new Response(new Uint8Array([1, 2, 3])),
"/api/organizations/org_a1b2c3d4e5f6g7h8i9j0k1l2/documents/doc_1": () =>
json({
document: {
id: "doc_1",
name: "Blood test.pdf",
originalName: "blood.pdf",
documentDate: "2025-02-01",
createdAt: "2025-03-04T09:00:00.000Z",
},
}),
});
const client = sourceClient(PAPRA);
await expect(client.document("doc_1")).resolves.toEqual({
title: "Blood test",
date: "2025-02-01",
filename: "blood.pdf",
});
const file = await client.download("doc_1", 1024);
expect([...file.bytes]).toEqual([1, 2, 3]);
});
});
@@ -0,0 +1,110 @@
import { afterEach, describe, expect, it, vi } from "vitest";
import {
configuredDocumentSourceOrigins,
documentSourcesAvailable,
evaluateSourceBaseUrl,
listedDocumentSourceOrigins,
} from "../origins";
const ORIGINAL = process.env.DOCUMENT_SOURCE_ORIGINS;
afterEach(() => {
if (ORIGINAL === undefined) delete process.env.DOCUMENT_SOURCE_ORIGINS;
else process.env.DOCUMENT_SOURCE_ORIGINS = ORIGINAL;
vi.restoreAllMocks();
});
describe("DOCUMENT_SOURCE_ORIGINS", () => {
it("is off when unset or empty", () => {
delete process.env.DOCUMENT_SOURCE_ORIGINS;
expect(documentSourcesAvailable()).toBe(false);
process.env.DOCUMENT_SOURCE_ORIGINS = " ";
expect(documentSourcesAvailable()).toBe(false);
});
it("keeps valid exact origins and drops malformed ones with a redacted warning", () => {
const warn = vi.spyOn(console, "warn").mockImplementation(() => {});
process.env.DOCUMENT_SOURCE_ORIGINS =
"https://paperless.example.com, http://papra.lan:1221, https://user:secret@bad.example/x?token=abc, http://169.254.169.254";
const origins = configuredDocumentSourceOrigins();
expect([...origins].sort()).toEqual([
"http://papra.lan:1221",
"https://paperless.example.com",
]);
expect(documentSourcesAvailable()).toBe(true);
const logged = warn.mock.calls.map((c) => String(c[0])).join("\n");
expect(logged).toContain("DOCUMENT_SOURCE_ORIGINS");
expect(logged).not.toContain("secret");
expect(logged).not.toContain("token=abc");
});
it("lists the origins for the settings card", () => {
process.env.DOCUMENT_SOURCE_ORIGINS =
"https://paperless.example.com,http://papra.lan:1221";
expect(listedDocumentSourceOrigins()).toEqual([
"http://papra.lan:1221",
"https://paperless.example.com",
]);
});
});
describe("evaluateSourceBaseUrl", () => {
const listed = new Set(["https://paperless.example.com", "http://10.0.0.5"]);
it("accepts a listed origin and keeps a base path without the trailing slash", () => {
expect(
evaluateSourceBaseUrl("https://paperless.example.com/paperless/", listed),
).toEqual({
ok: true,
origin: "https://paperless.example.com",
baseUrl: "https://paperless.example.com/paperless",
});
expect(evaluateSourceBaseUrl("http://10.0.0.5", listed)).toEqual({
ok: true,
origin: "http://10.0.0.5",
baseUrl: "http://10.0.0.5",
});
});
it("refuses an origin that is not listed, public or private", () => {
for (const url of [
"https://example.org",
"http://10.0.0.6",
"http://192.168.1.10:8000",
"http://169.254.169.254",
"http://localhost:8000",
]) {
expect(evaluateSourceBaseUrl(url, listed), url).toEqual({
ok: false,
reason: "notAllowed",
});
}
});
it("does not turn a listed host into a suffix, sibling port or other scheme grant", () => {
for (const url of [
"https://evil.paperless.example.com",
"https://paperless.example.com:8443",
"http://paperless.example.com",
"https://paperless.example.com.evil.test",
]) {
expect(evaluateSourceBaseUrl(url, listed).ok, url).toBe(false);
}
});
it("refuses userinfo, a query, a fragment and non-http schemes", () => {
for (const url of [
"https://user:pass@paperless.example.com",
"https://paperless.example.com/?token=x",
"https://paperless.example.com/#x",
"ftp://paperless.example.com",
"not a url",
]) {
expect(evaluateSourceBaseUrl(url, listed), url).toEqual({
ok: false,
reason: "invalid",
});
}
});
});
+533
View File
@@ -0,0 +1,533 @@
/**
* The two source adapters the document picker speaks: Paperless-ngx and Papra.
*
* Each turns the picker's four questions (does this connection work, which
* tags exist, which documents match, give me this one) into the source's own
* REST calls, and each answer into the picker's minimal shapes. Everything
* leaves through `./http.ts`, so the origin pin, the redirect refusal, the
* timeouts and the byte caps are not something an adapter can forget.
*
* Both adapters re-check tag and date filters on what comes back. A server
* that ignores a parameter (an older Paperless, a Papra whose search syntax
* changed) must not widen a result into documents the person filtered out;
* the import script makes the same check for the same reason.
*
* API versions: Paperless-ngx is asked for API version 9 (Paperless-ngx 2.16
* and later, the version where `created` became a plain date; it answers 406
* to a version it does not speak). Papra's endpoints are the documented
* organization-scoped ones (`/api/organizations/{id}/documents`, `/tags`,
* `/documents/{id}`, `/documents/{id}/file`).
*/
import { DOCUMENT_TITLE_MAX } from "@/lib/validations/inbound-documents";
import {
DocumentSourceError,
sourceDownload,
sourceJson,
type SourceTarget,
} from "./http";
import {
DOCUMENT_PICKER_MAX_TAGS,
DOCUMENT_PICKER_PAGE_SIZE,
isSourceDocumentId,
PAPRA_ORG_ID,
type DocumentPickerSystem,
type DocumentSourceErrorCode,
type DocumentSourceTagDto,
} from "./types";
/** Paperless-ngx API version the picker speaks (Paperless-ngx 2.16+). */
export const PAPERLESS_API_VERSION = 9;
export interface SourceSearchParams {
/** Name to search for; empty lists everything. */
q: string;
/** A tag id as the source names it. */
tagId: string | null;
/** YYYY-MM-DD, inclusive. */
from: string | null;
to: string | null;
/** 1-based. */
page: number;
}
export interface SourceListItem {
sourceId: string;
title: string;
date: string | null;
tags: string[];
sizeBytes: number | null;
}
export interface SourceDocumentMeta {
title: string;
date: string | null;
filename: string | null;
}
export interface SourceClient {
readonly system: DocumentPickerSystem;
/**
* The instance this client talks to, as the normalised origin every source
* key it produces carries (`sourceInstance`).
*/
readonly instance: string;
test(): Promise<void>;
tags(): Promise<DocumentSourceTagDto[]>;
search(
params: SourceSearchParams,
): Promise<{ items: SourceListItem[]; hasMore: boolean }>;
document(sourceId: string): Promise<SourceDocumentMeta>;
download(
sourceId: string,
maxFileBytes: number,
): Promise<{ bytes: Uint8Array; filename: string | null }>;
}
export interface SourceConnection {
system: DocumentPickerSystem;
origin: string;
baseUrl: string;
organizationId: string | null;
token: string;
}
/** Paperless tags are read this many per page. */
const TAG_PAGE_SIZE = 250;
/** Refuse an id that is not one the system issues, before it reaches a URL. */
function requireId(system: DocumentPickerSystem, id: string): void {
if (!isSourceDocumentId(system, id)) {
throw new DocumentSourceError("notFound");
}
}
// ── Defensive readers ──────────────────────────────────────────────────────
type Json = Record<string, unknown>;
function asRecord(value: unknown): Json | null {
return value && typeof value === "object" && !Array.isArray(value)
? (value as Json)
: null;
}
function asArray(value: unknown): unknown[] {
return Array.isArray(value) ? value : [];
}
function asText(value: unknown): string | null {
if (typeof value === "string") return value;
if (typeof value === "number" && Number.isFinite(value)) return String(value);
return null;
}
/** The body is not the shape the API documents: most often a base address
* that points at the web app instead of the instance. */
function shapeError(): DocumentSourceError {
return new DocumentSourceError("badResponse");
}
/** YYYY-MM-DD from an ISO date or date-time, or null. */
export function dayOf(value: unknown): string | null {
if (typeof value !== "string") return null;
const day = value.slice(0, 10);
return /^\d{4}-\d{2}-\d{2}$/.test(day) ? day : null;
}
function stripExtension(name: string | null): string {
return name ? name.replace(/\.[A-Za-z0-9]{1,5}$/, "") : "";
}
/**
* A title that is not blank and fits the vault: the source's title, else its
* file name without the extension, else a placeholder naming the source id.
*/
export function titleOf(
title: string | null,
filename: string | null,
fallback: string,
): string {
const picked =
(title ?? "").trim() || stripExtension(filename).trim() || fallback;
return picked.slice(0, DOCUMENT_TITLE_MAX);
}
/** Is `day` inside the inclusive range? A document without a date is kept
* only when no range was asked for. */
export function withinRange(
day: string | null,
from: string | null,
to: string | null,
): boolean {
if (!from && !to) return true;
if (!day) return false;
if (from && day < from) return false;
if (to && day > to) return false;
return true;
}
// ── Paperless-ngx ──────────────────────────────────────────────────────────
function paperless(connection: SourceConnection): SourceClient {
const target: SourceTarget = {
origin: connection.origin,
baseUrl: connection.baseUrl,
headers: {
Authorization: `Token ${connection.token}`,
Accept: `application/json; version=${PAPERLESS_API_VERSION}`,
},
};
const download: SourceTarget = {
...target,
headers: { Authorization: `Token ${connection.token}` },
};
/** A server that says which API version it speaks, and it is too old. */
function checkVersion(response: Response): void {
const raw = response.headers.get("x-api-version");
if (raw === null) return;
const version = Number(raw);
if (Number.isFinite(version) && version < PAPERLESS_API_VERSION) {
throw new DocumentSourceError("versionTooOld", response.status);
}
}
async function tagMap(): Promise<Map<string, string>> {
const tags = await listTags();
return new Map(tags.map((tag) => [tag.id, tag.name]));
}
async function listTags(): Promise<DocumentSourceTagDto[]> {
// Paged by number rather than by following `next`: the link is the
// server's to write, the page number is ours.
const tags: DocumentSourceTagDto[] = [];
for (let pageNo = 1; tags.length < DOCUMENT_PICKER_MAX_TAGS; pageNo += 1) {
const query = new URLSearchParams({
page: String(pageNo),
page_size: String(TAG_PAGE_SIZE),
ordering: "name",
});
const { body, response } = await sourceJson(
target,
`/api/tags/?${query}`,
);
checkVersion(response);
const page = asRecord(body);
if (!page) throw shapeError();
for (const raw of asArray(page.results)) {
const tag = asRecord(raw);
const id = asText(tag?.id);
const name = asText(tag?.name);
if (id && name) tags.push({ id, name });
}
if (typeof page.next !== "string" || !page.next) break;
}
return tags.slice(0, DOCUMENT_PICKER_MAX_TAGS);
}
return {
system: "PAPERLESS",
instance: connection.origin,
async test() {
const { body, response } = await sourceJson(
target,
"/api/documents/?page_size=1&fields=id",
);
checkVersion(response);
if (!asRecord(body)) throw shapeError();
},
tags: listTags,
async search(params) {
const query = new URLSearchParams({
page: String(params.page),
page_size: String(DOCUMENT_PICKER_PAGE_SIZE),
ordering: "-created",
truncate_content: "true",
fields: "id,title,created,added,tags,original_file_name",
});
if (params.q) query.set("title__icontains", params.q);
if (params.tagId) query.set("tags__id__all", params.tagId);
// API version 9 made `created` a plain date, so the range filters on
// it directly (`created__gte` / `created__lte`), both ends inclusive.
if (params.from) query.set("created__gte", params.from);
if (params.to) query.set("created__lte", params.to);
let page: Json | null;
let names: Map<string, string>;
try {
const [list, tags] = await Promise.all([
sourceJson(target, `/api/documents/?${query}`),
tagMap(),
]);
checkVersion(list.response);
page = asRecord(list.body);
names = tags;
} catch (err) {
// Paperless answers a page past the end with 404 ("Invalid page"):
// there is simply nothing more.
if (err instanceof DocumentSourceError && err.code === "notFound") {
return { items: [], hasMore: false };
}
throw err;
}
if (!page) throw shapeError();
const items: SourceListItem[] = [];
for (const raw of asArray(page.results)) {
const doc = asRecord(raw);
const id = asText(doc?.id);
if (!doc || !id || !isSourceDocumentId("PAPERLESS", id)) continue;
const tagIds = asArray(doc.tags)
.map(asText)
.filter((t): t is string => t !== null);
if (params.tagId && !tagIds.includes(params.tagId)) continue;
const date = dayOf(doc.created) ?? dayOf(doc.added);
if (!withinRange(date, params.from, params.to)) continue;
items.push({
sourceId: id,
title: titleOf(
asText(doc.title),
asText(doc.original_file_name),
`Paperless ${id}`,
),
date,
tags: tagIds
.map((tagId) => names.get(tagId))
.filter((n): n is string => typeof n === "string"),
sizeBytes: null,
});
}
return { items, hasMore: typeof page.next === "string" && !!page.next };
},
async document(sourceId) {
requireId("PAPERLESS", sourceId);
const { body, response } = await sourceJson(
target,
`/api/documents/${encodeURIComponent(sourceId)}/?fields=id,title,created,added,original_file_name`,
);
checkVersion(response);
const doc = asRecord(body);
if (!doc) throw shapeError();
const filename = asText(doc.original_file_name);
return {
title: titleOf(asText(doc.title), filename, `Paperless ${sourceId}`),
date: dayOf(doc.created) ?? dayOf(doc.added),
filename,
};
},
async download(sourceId, maxFileBytes) {
requireId("PAPERLESS", sourceId);
// The original, not the archived PDF/A: the same bytes a Paperless
// workflow or the import script sends, so they dedup against each other.
return sourceDownload(
download,
`/api/documents/${encodeURIComponent(sourceId)}/download/?original=true`,
maxFileBytes,
);
},
};
}
// ── Papra ──────────────────────────────────────────────────────────────────
/**
* Papra's error bodies carry a machine code. Two of them mean "no such
* organization" rather than "wrong key": a valid organization id the key's
* user is not a member of (403 `user.not_in_organization`), and one Papra
* cannot parse (400 `server.invalid_request.params`).
*/
function papraError(
status: number,
code: string | null,
): DocumentSourceErrorCode | null {
if (status === 403 && code === "user.not_in_organization") return "notFound";
if (status === 400 && code === "server.invalid_request.params") {
return "notFound";
}
return null;
}
/** Papra's search syntax quotes a tag name that holds a space. */
function papraTagTerm(name: string): string {
return /[\s"]/.test(name) ? `tag:${JSON.stringify(name)}` : `tag:${name}`;
}
function papra(connection: SourceConnection): SourceClient {
if (
!connection.organizationId ||
!PAPRA_ORG_ID.test(connection.organizationId)
) {
throw new DocumentSourceError("notFound");
}
const org = `/api/organizations/${encodeURIComponent(connection.organizationId)}`;
const target: SourceTarget = {
origin: connection.origin,
baseUrl: connection.baseUrl,
headers: {
Authorization: `Bearer ${connection.token}`,
Accept: "application/json",
},
classify: papraError,
};
const download: SourceTarget = {
...target,
headers: { Authorization: `Bearer ${connection.token}` },
};
async function listTags(): Promise<DocumentSourceTagDto[]> {
const { body } = await sourceJson(target, `${org}/tags`);
const page = asRecord(body);
if (!page) throw shapeError();
const tags: DocumentSourceTagDto[] = [];
for (const raw of asArray(page.tags)) {
const tag = asRecord(raw);
const id = asText(tag?.id);
const name = asText(tag?.name);
if (id && name) tags.push({ id, name });
}
return tags
.sort((a, b) => a.name.localeCompare(b.name))
.slice(0, DOCUMENT_PICKER_MAX_TAGS);
}
return {
system: "PAPRA",
instance: connection.origin,
async test() {
const { body } = await sourceJson(
target,
`${org}/documents?pageIndex=0&pageSize=1`,
);
if (!asRecord(body)) throw shapeError();
// The tag filter needs `tags:read`; a key without it would save green
// and then fail on the first search.
try {
await sourceJson(target, `${org}/tags`);
} catch (err) {
if (err instanceof DocumentSourceError && err.code === "authRefused") {
throw new DocumentSourceError(
"permissionMissing",
err.upstreamStatus,
);
}
throw err;
}
},
tags: listTags,
async search(params) {
// Each word quoted: Papra reads a leading `-` as a negation and a `:` as
// a field, and a quoted word still matches as a prefix.
const terms: string[] = params.q
.split(/\s+/)
.map((word) => word.replace(/"/g, ""))
.filter((word) => word.length > 0)
.map((word) => `"${word}"`);
if (params.tagId) {
const tag = (await listTags()).find((t) => t.id === params.tagId);
// A tag that no longer exists matches nothing, rather than silently
// dropping the filter and listing everything.
if (!tag) return { items: [], hasMore: false };
terms.push(papraTagTerm(tag.name));
}
// The range goes to Papra as its own `date:` filter; the check on each
// row below stays as the net for a version that reads it differently.
if (params.from) terms.push(`date:>=${params.from}`);
if (params.to) terms.push(`date:<=${params.to}`);
const pageIndex = params.page - 1;
const query = new URLSearchParams({
pageIndex: String(pageIndex),
pageSize: String(DOCUMENT_PICKER_PAGE_SIZE),
});
if (terms.length > 0) query.set("searchQuery", terms.join(" "));
const { body } = await sourceJson(target, `${org}/documents?${query}`);
const page = asRecord(body);
if (!page) throw shapeError();
const documents = asArray(page.documents);
const items: SourceListItem[] = [];
for (const raw of documents) {
const doc = asRecord(raw);
const id = asText(doc?.id);
if (!doc || !id || !isSourceDocumentId("PAPRA", id)) continue;
const tags = asArray(doc.tags)
.map(asRecord)
.filter((t): t is Json => t !== null);
if (params.tagId && !tags.some((t) => asText(t.id) === params.tagId)) {
continue;
}
// The row shows the document date only, the field Papra's `date:`
// filter reads, so the list and the filter agree: an undated document
// shows no date and falls out of any range. The check below is the net
// for a server that ignores the filter. The upload day is used only
// on import, where a filing date is needed (see `document()`).
const date = dayOf(doc.documentDate);
if (!withinRange(date, params.from, params.to)) continue;
const size = doc.originalSize;
items.push({
sourceId: id,
title: titleOf(
stripExtension(asText(doc.name)),
asText(doc.originalName),
`Papra ${id}`,
),
date,
tags: tags
.map((t) => asText(t.name))
.filter((n): n is string => n !== null),
sizeBytes: typeof size === "number" && size >= 0 ? size : null,
});
}
const total =
typeof page.documentsCount === "number" ? page.documentsCount : null;
const hasMore =
total !== null
? (pageIndex + 1) * DOCUMENT_PICKER_PAGE_SIZE < total
: documents.length >= DOCUMENT_PICKER_PAGE_SIZE;
return { items, hasMore };
},
async document(sourceId) {
requireId("PAPRA", sourceId);
const { body } = await sourceJson(
target,
`${org}/documents/${encodeURIComponent(sourceId)}`,
);
const doc = asRecord(asRecord(body)?.document);
if (!doc) throw shapeError();
const filename = asText(doc.originalName) ?? asText(doc.name);
return {
title: titleOf(
stripExtension(asText(doc.name)),
filename,
`Papra ${sourceId}`,
),
date: dayOf(doc.documentDate) ?? dayOf(doc.createdAt),
filename,
};
},
async download(sourceId, maxFileBytes) {
requireId("PAPRA", sourceId);
return sourceDownload(
download,
`${org}/documents/${encodeURIComponent(sourceId)}/file`,
maxFileBytes,
);
},
};
}
export function sourceClient(connection: SourceConnection): SourceClient {
return connection.system === "PAPERLESS"
? paperless(connection)
: papra(connection);
}
+117
View File
@@ -0,0 +1,117 @@
/**
* Reading a person's document-source connection, and the one place its token
* is decrypted.
*
* {@link toConnectionDto} is the only shape a connection leaves the server in,
* and it has no field that could carry the token: `hasToken` is the literal
* `true`. {@link openConnection} decrypts the token for exactly one request to
* the source and re-checks the stored origin against the operator's list on
* the way, so a connection whose origin was removed from
* `DOCUMENT_SOURCE_ORIGINS` answers `originNotAllowed` instead of dialling.
*/
import { decrypt } from "@/lib/crypto";
import { prisma } from "@/lib/db";
import { sourceClient, type SourceClient } from "./clients";
import { DocumentSourceError } from "./http";
import { evaluateSourceBaseUrl } from "./origins";
import type {
DocumentPickerSystem,
DocumentSourceConnectionDto,
} from "./types";
/** The columns a DTO or a request needs; never the whole row by accident. */
const CONNECTION_SELECT = {
system: true,
baseUrl: true,
organizationId: true,
tokenEncrypted: true,
lastVerifiedAt: true,
} as const;
export interface StoredConnection {
system: string;
baseUrl: string;
organizationId: string | null;
tokenEncrypted: string;
lastVerifiedAt: Date | null;
}
export function toConnectionDto(
row: Omit<StoredConnection, "tokenEncrypted">,
): DocumentSourceConnectionDto {
return {
system: row.system as DocumentPickerSystem,
baseUrl: row.baseUrl,
organizationId: row.organizationId,
hasToken: true,
lastVerifiedAt: row.lastVerifiedAt?.toISOString() ?? null,
originAllowed: evaluateSourceBaseUrl(row.baseUrl).ok,
};
}
export function listConnections(
userId: string,
): Promise<Omit<StoredConnection, "tokenEncrypted">[]> {
return prisma.documentSourceConnection.findMany({
where: { userId },
select: {
system: true,
baseUrl: true,
organizationId: true,
lastVerifiedAt: true,
},
orderBy: { system: "asc" },
});
}
export function loadConnection(
userId: string,
system: DocumentPickerSystem,
): Promise<StoredConnection | null> {
return prisma.documentSourceConnection.findUnique({
where: { userId_system: { userId, system } },
select: CONNECTION_SELECT,
});
}
/**
* A client for one of the person's stored connections, or the reason there is
* none. Throws {@link DocumentSourceError}: `notConnected` when nothing is
* stored for the system, `originNotAllowed` when the operator no longer lists
* the stored origin.
*/
export async function openConnection(
userId: string,
system: DocumentPickerSystem,
): Promise<SourceClient> {
const row = await loadConnection(userId, system);
if (!row) throw new DocumentSourceError("notConnected");
return clientFor(row);
}
/** A client for a row already in hand, with the origin re-checked. */
export function clientFor(row: StoredConnection): SourceClient {
const verdict = evaluateSourceBaseUrl(row.baseUrl);
if (!verdict.ok) throw new DocumentSourceError("originNotAllowed");
return sourceClient({
system: row.system as DocumentPickerSystem,
origin: verdict.origin,
baseUrl: verdict.baseUrl,
organizationId: row.organizationId,
token: decrypt(row.tokenEncrypted),
});
}
/** Stamp a successful test. */
export async function markVerified(
userId: string,
system: DocumentPickerSystem,
): Promise<Date> {
const at = new Date();
await prisma.documentSourceConnection.updateMany({
where: { userId, system },
data: { lastVerifiedAt: at },
});
return at;
}
+250
View File
@@ -0,0 +1,250 @@
/**
* The document picker's one way to the network.
*
* Every request to a Paperless-ngx or Papra instance goes through
* {@link sourceRequest}, which goes through `safeFetch` with the connection's
* origin as `operatorApprovedPrivateOrigin`:
*
* - the target must have exactly that origin (a path, query or `next` link
* that names another host is refused before a socket opens);
* - redirects are never followed; a 3xx is reported as `redirected`, so a
* source cannot bounce the request and its `Authorization` header to
* another host;
* - the connect goes through the pinned operator-approved dispatcher, which
* refuses link-local, metadata and unspecified addresses at dial time even
* for a listed name;
* - every call has a timeout, and every body is read with a byte cap.
*
* The origin handed in is the one the caller just re-checked against
* `DOCUMENT_SOURCE_ORIGINS`. Nothing here decides trust; it only refuses to
* go anywhere else.
*/
import { safeFetch, SafeFetchError } from "@/lib/safe-fetch";
import { BodyTooLargeError, readBoundedBody } from "@/lib/labs/ocr-upload";
import { annotate } from "@/lib/logging/context";
import type { DocumentSourceErrorCode } from "./types";
/** JSON bodies larger than this are not a search page; refuse them. */
export const SOURCE_JSON_MAX_BYTES = 1024 * 1024;
export const SOURCE_JSON_TIMEOUT_MS = 10_000;
export const SOURCE_DOWNLOAD_TIMEOUT_MS = 60_000;
/**
* A request to a source ended somewhere other than a usable answer. The code
* is what the client translates; `upstreamStatus` is the only thing about the
* source's answer that is passed on. Its body never is: an error page can
* echo the request's credentials or internal paths.
*/
export class DocumentSourceError extends Error {
readonly code: DocumentSourceErrorCode;
readonly upstreamStatus?: number;
constructor(code: DocumentSourceErrorCode, upstreamStatus?: number) {
super(`document source: ${code}`);
this.name = "DocumentSourceError";
this.code = code;
if (upstreamStatus !== undefined) this.upstreamStatus = upstreamStatus;
}
}
/** The document is larger than the vault accepts; carries the cap. */
export class DocumentSourceTooLargeError extends Error {
constructor(readonly maxFileBytes: number) {
super("document source: file larger than the vault accepts");
this.name = "DocumentSourceTooLargeError";
}
}
export interface SourceTarget {
/** The listed origin, exactly as `DOCUMENT_SOURCE_ORIGINS` holds it. */
origin: string;
/** Origin plus the optional base path, no trailing slash. */
baseUrl: string;
headers: Record<string, string>;
/** How this source's error bodies read, when they say more than a status. */
classify?: ErrorClassifier;
}
async function sourceRequest(
target: SourceTarget,
path: string,
timeoutMs: number,
): Promise<Response> {
const url = `${target.baseUrl}${path}`;
let response: Response;
try {
response = await safeFetch(
url,
{ method: "GET", headers: target.headers, redirect: "manual" },
{ operatorApprovedPrivateOrigin: target.origin, timeoutMs },
);
} catch (err) {
// A refused destination, a timeout and a dead host all read the same to
// the person: the source could not be reached from here. The operator's
// wide event keeps the kind.
annotate({
meta: {
sourceFailure: err instanceof SafeFetchError ? err.kind : "network",
},
});
throw new DocumentSourceError("unreachable");
}
if (response.status >= 300 && response.status < 400) {
await discard(response);
throw new DocumentSourceError("redirected", response.status);
}
return response;
}
async function discard(response: Response): Promise<void> {
try {
await response.body?.cancel();
} catch {
// nothing to release
}
}
/**
* A source-specific reading of an error answer: given the status and the
* machine code in the body (if any), the error to report, or null for the
* generic mapping. The body itself is never passed on.
*/
export type ErrorClassifier = (
status: number,
code: string | null,
) => DocumentSourceErrorCode | null;
/** Longest error body read to find its machine code. */
const ERROR_BODY_MAX_BYTES = 4 * 1024;
/** The `code` (or `error.code`) of a JSON error body, if there is one. */
async function errorCodeOf(response: Response): Promise<string | null> {
try {
const bytes = await readBoundedBody(response.body, ERROR_BODY_MAX_BYTES);
const body = JSON.parse(new TextDecoder().decode(bytes)) as unknown;
if (!body || typeof body !== "object") return null;
const record = body as Record<string, unknown>;
const nested = record.error as Record<string, unknown> | undefined;
const code =
typeof record.code === "string"
? record.code
: nested &&
typeof nested === "object" &&
typeof nested.code === "string"
? nested.code
: null;
return code && code.length <= 100 ? code : null;
} catch {
return null;
}
}
/** Map a non-2xx answer to the error the person sees. */
export async function failFor(
response: Response,
notFound: DocumentSourceErrorCode = "notFound",
classify?: ErrorClassifier,
): Promise<never> {
const status = response.status;
if (classify) {
const specific = classify(status, await errorCodeOf(response));
if (specific) throw new DocumentSourceError(specific, status);
} else {
await discard(response);
}
if (status === 401 || status === 403) {
throw new DocumentSourceError("authRefused", status);
}
if (status === 404) throw new DocumentSourceError(notFound, status);
// Paperless-ngx answers 406 to an API version it no longer (or not yet)
// speaks.
if (status === 406) throw new DocumentSourceError("versionTooOld", status);
throw new DocumentSourceError("badResponse", status);
}
/** GET a JSON document, bounded in time and size. */
export async function sourceJson(
target: SourceTarget,
path: string,
options: {
notFound?: DocumentSourceErrorCode;
classify?: ErrorClassifier;
} = {},
): Promise<{ body: unknown; response: Response }> {
const response = await sourceRequest(target, path, SOURCE_JSON_TIMEOUT_MS);
if (!response.ok) {
return failFor(
response,
options.notFound,
options.classify ?? target.classify,
);
}
const declared = Number(response.headers.get("content-length") ?? 0);
if (declared > SOURCE_JSON_MAX_BYTES) {
await discard(response);
throw new DocumentSourceError("badResponse", response.status);
}
let bytes: Uint8Array;
try {
bytes = await readBoundedBody(response.body, SOURCE_JSON_MAX_BYTES);
} catch {
throw new DocumentSourceError("badResponse", response.status);
}
try {
return {
body: JSON.parse(new TextDecoder().decode(bytes)) as unknown,
response,
};
} catch {
// A login page or a proxy's HTML error instead of the API: most often a
// base address that points at the web app rather than its API root.
throw new DocumentSourceError("badResponse", response.status);
}
}
/** GET a file, bounded by the vault's own per-file cap. */
export async function sourceDownload(
target: SourceTarget,
path: string,
maxFileBytes: number,
): Promise<{ bytes: Uint8Array; filename: string | null }> {
const response = await sourceRequest(
target,
path,
SOURCE_DOWNLOAD_TIMEOUT_MS,
);
if (!response.ok) return failFor(response, "notFound", target.classify);
const declared = Number(response.headers.get("content-length") ?? 0);
if (declared > maxFileBytes) {
await discard(response);
throw new DocumentSourceTooLargeError(maxFileBytes);
}
try {
const bytes = await readBoundedBody(response.body, maxFileBytes);
return {
bytes,
filename: filenameFrom(response.headers.get("content-disposition")),
};
} catch (err) {
if (err instanceof BodyTooLargeError) {
throw new DocumentSourceTooLargeError(maxFileBytes);
}
throw new DocumentSourceError("unreachable");
}
}
/** The file name a download names, if it names one. */
export function filenameFrom(header: string | null): string | null {
if (!header) return null;
const star = /filename\*=(?:UTF-8'')?([^;]+)/i.exec(header);
if (star) {
try {
return decodeURIComponent(star[1].trim().replace(/^"|"$/g, ""));
} catch {
// fall through to the plain form
}
}
const plain = /filename="?([^";]+)"?/i.exec(header);
return plain ? plain[1] : null;
}
+95
View File
@@ -0,0 +1,95 @@
/**
* `DOCUMENT_SOURCE_ORIGINS` (#1038) — the operator grant that lets the
* document picker reach a Paperless-ngx or Papra instance.
*
* DOCUMENT_SOURCE_ORIGINS="https://paperless.example.com,http://papra.lan:1221"
*
* The grammar is the one every private-origin list shares
* (`src/lib/private-origin-policy.ts`): exact `scheme://host[:port]` origins,
* no path, query, credentials, wildcard, CIDR or suffix. A malformed entry is
* logged once, reduced to scheme and host, and grants nothing; the valid
* entries beside it keep working.
*
* Unlike the notification list, this one is the WHOLE trust decision, not an
* exception on top of a public-host default. An origin that is not listed is
* refused even when it is a public address, and with the variable unset the
* picker does not exist: the routes answer 404 and the client renders no
* button. A person can only ever point their connection at a system the
* operator named. Every request to a listed origin still goes through
* `safeFetch` with the origin pinned, redirects forbidden, and the dial-time
* floor that refuses link-local, metadata and unspecified addresses even for
* a listed name.
*
* Read directly from `process.env` so the compose-whitelist guard sees it,
* parsed once per distinct value, and re-read on every use: an operator who
* removes an origin cuts off the stored connections that name it on their
* next request.
*/
import {
parsePrivateOrigins,
redactGrantEntry,
} from "@/lib/private-origin-policy";
const ENV_NAME = "DOCUMENT_SOURCE_ORIGINS";
let cachedRaw: string | undefined;
let cachedOrigins: ReadonlySet<string> = new Set();
export function configuredDocumentSourceOrigins(): ReadonlySet<string> {
const raw = process.env.DOCUMENT_SOURCE_ORIGINS;
if (raw === cachedRaw) return cachedOrigins;
cachedOrigins = parsePrivateOrigins(raw, (redactedEntry, reason) => {
console.warn(`${ENV_NAME}: ignoring entry "${redactedEntry}": ${reason}`);
});
cachedRaw = raw;
return cachedOrigins;
}
/** Is the picker switched on for this instance at all? */
export function documentSourcesAvailable(): boolean {
return configuredDocumentSourceOrigins().size > 0;
}
/** Longest base URL a connection may store. */
export const DOCUMENT_SOURCE_BASE_URL_MAX = 2048;
export type SourceBaseVerdict =
| { ok: true; origin: string; baseUrl: string }
| { ok: false; reason: "invalid" | "notAllowed" };
/**
* Validate a base URL a person typed against the operator's list.
*
* The base may carry a path (a Paperless served under `/paperless`); only its
* origin is compared, and it must be listed exactly. Userinfo, a query and a
* fragment are refused: none of them belongs in a base address, and each is
* where a pasted credential would sit. Trailing slashes are trimmed so the
* client can append `/api/...` without doubling one.
*/
export function evaluateSourceBaseUrl(
value: string,
origins: ReadonlySet<string> = configuredDocumentSourceOrigins(),
): SourceBaseVerdict {
let url: URL;
try {
url = new URL(value.trim());
} catch {
return { ok: false, reason: "invalid" };
}
if (url.protocol !== "http:" && url.protocol !== "https:") {
return { ok: false, reason: "invalid" };
}
if (url.username || url.password || url.search || url.hash) {
return { ok: false, reason: "invalid" };
}
if (!origins.has(url.origin)) return { ok: false, reason: "notAllowed" };
const path = url.pathname.replace(/\/+$/, "");
return { ok: true, origin: url.origin, baseUrl: `${url.origin}${path}` };
}
/** The listed origins as the settings card may show them. */
export function listedDocumentSourceOrigins(): string[] {
return [...configuredDocumentSourceOrigins()]
.map((origin) => redactGrantEntry(origin))
.sort();
}
+165
View File
@@ -0,0 +1,165 @@
/**
* What every document-source route does before its own work, and how a source
* failure becomes a response.
*
* The admission rule, with {@link admitDocumentSourceCaller} as its second
* half:
*
* - the route's own `requireAuth()` with no scope: a session, or a `["*"]` token; a narrow
* token is refused, and so is any request acting on somebody else's record
* (403 `sharing.not_permitted`). A delegate, whatever their grant, and a
* guardian acting for a managed record never reach a connection: it is the
* owner's credential to a system outside HealthLog, and driving it would
* read the owner's whole archive there, which no grant covers;
* - then the transport must be the cookie. A `["*"]` Bearer (the native
* client, a script holding a login token) is refused: the picker is a
* browser surface, and a credential for another system is managed only
* from one;
* - the documents module must be on, and the operator must have listed at
* least one origin in `DOCUMENT_SOURCE_ORIGINS`. Without the list the
* routes answer 404 before reading a row.
*/
import type { AuthContext } from "@/lib/api-handler";
import { apiError } from "@/lib/api-response";
import { requireModuleEnabled } from "@/lib/modules/gate";
import { checkRateLimit, type RateLimitResult } from "@/lib/rate-limit";
import { DocumentSourceError, DocumentSourceTooLargeError } from "./http";
import { documentSourcesAvailable } from "./origins";
import {
documentSourceErrorCode,
systemFromSlug,
type DocumentPickerSystem,
type DocumentSourceErrorCode,
} from "./types";
/**
* Admit a caller the route has already resolved with a bare `requireAuth()`.
*
* Each route calls `requireAuth()` itself, in its own module, so the sharing
* guard's scan sees every one of them as a bare, owner-only route rather than
* a helper it cannot follow. This adds the rest: a cookie transport, the
* documents module and the operator's list. The status read and the delete
* pass `needsList: false, needsModule: false`: they touch only the person's
* own row and never the network.
*/
export async function admitDocumentSourceCaller(
auth: AuthContext,
options: { needsList?: boolean; needsModule?: boolean } = {},
): Promise<Response | null> {
if (auth.authMethod !== "cookie") {
return apiError(
"Document archives are managed from a browser session.",
403,
{ errorCode: "documents.sources.browserOnly" },
);
}
// Reading and removing one's own connection needs neither the module nor
// the list: a person who switched Documents off, or whose operator removed
// the list, must still be able to see and delete the stored token.
if (options.needsModule !== false) {
const gate = await requireModuleEnabled(auth.user.id, "inboundDocuments");
if (!gate.enabled) return gate.response;
}
if (options.needsList !== false && !documentSourcesAvailable()) {
return sourceErrorResponse("unavailable");
}
return null;
}
/** The `{system}` path segment, or a 404 for anything else. */
export async function systemParam(
params: Promise<{ system: string }>,
): Promise<DocumentPickerSystem | Response> {
const { system } = await params;
const resolved = systemFromSlug(system);
return resolved ?? apiError("Unknown document system.", 404);
}
const STATUS: Record<DocumentSourceErrorCode, number> = {
unavailable: 404,
notConnected: 404,
originNotAllowed: 422,
unreachable: 502,
redirected: 502,
authRefused: 502,
permissionMissing: 502,
versionTooOld: 502,
badResponse: 502,
notFound: 404,
linkTargetNotFound: 404,
rateLimited: 429,
};
const MESSAGE: Record<DocumentSourceErrorCode, string> = {
unavailable: "Document archives are not enabled on this server.",
notConnected: "This document archive is not connected.",
originNotAllowed:
"This address is not on the server's list of allowed document archives.",
unreachable: "The document archive could not be reached.",
redirected:
"The document archive redirected the request. Use the address it redirects to.",
authRefused: "The document archive refused the API token.",
permissionMissing:
"The API key cannot read the archive's tags. Give it the tags:read permission.",
versionTooOld: "The document archive's version is too old.",
badResponse: "The document archive gave an answer HealthLog cannot read.",
notFound: "The document archive has no such document or organization.",
linkTargetNotFound: "The record to link to was not found.",
rateLimited: "Too many requests. Try again later.",
};
export function sourceErrorResponse(
code: DocumentSourceErrorCode,
upstreamStatus?: number,
headers?: Record<string, string>,
): Response {
return apiError(MESSAGE[code], STATUS[code], {
errorCode: documentSourceErrorCode(code),
...(upstreamStatus !== undefined ? { upstreamStatus } : {}),
...(headers ? { headers } : {}),
});
}
/** Map a thrown source failure to its response; rethrow anything else. */
export function responseForSourceFailure(err: unknown): Response {
if (err instanceof DocumentSourceError) {
return sourceErrorResponse(err.code, err.upstreamStatus);
}
if (err instanceof DocumentSourceTooLargeError) {
return apiError("File is too large.", 413, {
errorCode: "documents.inbound.fileTooLarge",
reason: "fileTooLarge",
maxFileBytes: err.maxFileBytes,
});
}
throw err;
}
/** Saves and tests share one allowance: each one dials the source. */
const CONNECT_LIMIT_PER_MINUTE = 10;
export function checkConnectRateLimit(
userId: string,
): Promise<RateLimitResult> {
return checkRateLimit(
`documents-source-connect:${userId}`,
CONNECT_LIMIT_PER_MINUTE,
60_000,
);
}
/**
* Searches and tag reads per minute. Each one is a request to the source; a
* debounced search field sends one per pause in typing, which this leaves
* ample room for.
*/
const SEARCH_LIMIT_PER_MINUTE = 60;
export function checkSearchRateLimit(userId: string): Promise<RateLimitResult> {
return checkRateLimit(
`documents-source-search:${userId}`,
SEARCH_LIMIT_PER_MINUTE,
60_000,
);
}
+157
View File
@@ -0,0 +1,157 @@
/**
* Shared vocabulary of the document picker (#1038): which systems it speaks,
* the minimal shapes it hands to the client, and the errors a source can end
* in. A zero-dependency leaf so the client components, the OpenAPI registry
* and the routes import one list.
*/
/** The systems the picker can search and import from. */
export const DOCUMENT_PICKER_SYSTEMS = ["PAPERLESS", "PAPRA"] as const;
export type DocumentPickerSystem = (typeof DOCUMENT_PICKER_SYSTEMS)[number];
/** The lowercase path segment for each system (`/api/documents/sources/{slug}`). */
export const DOCUMENT_PICKER_SLUGS = {
paperless: "PAPERLESS",
papra: "PAPRA",
} as const satisfies Record<string, DocumentPickerSystem>;
export type DocumentPickerSlug = keyof typeof DOCUMENT_PICKER_SLUGS;
export function systemFromSlug(
slug: string | undefined,
): DocumentPickerSystem | null {
return slug && Object.hasOwn(DOCUMENT_PICKER_SLUGS, slug)
? DOCUMENT_PICKER_SLUGS[slug as DocumentPickerSlug]
: null;
}
export function slugForSystem(
system: DocumentPickerSystem,
): DocumentPickerSlug {
return system === "PAPERLESS" ? "paperless" : "papra";
}
/** Results per search page. */
export const DOCUMENT_PICKER_PAGE_SIZE = 25;
/** Highest page a search may ask for. */
export const DOCUMENT_PICKER_MAX_PAGE = 500;
/** Longest name query. */
export const DOCUMENT_PICKER_QUERY_MAX = 200;
/** Documents the sheet imports in one run. */
export const DOCUMENT_PICKER_MAX_SELECTION = 25;
/** Tags offered in the filter (read in pages of 250). */
export const DOCUMENT_PICKER_MAX_TAGS = 1000;
/** Paperless-ngx document ids are integers. */
export const PAPERLESS_ID = /^\d{1,18}$/;
/** Papra document ids: letters, digits, `_` and `-`. */
export const PAPRA_ID = /^[A-Za-z0-9_-]{1,128}$/;
/** Papra organization ids, as Papra issues them. */
export const PAPRA_ORG_ID = /^org_[a-z0-9]{24}$/;
/**
* Is `id` a document id `system` could have issued? Checked before an id is
* put into a request path, so nothing a request carries can add a path
* segment, a query or a `..` to the URL built from it.
*/
export function isSourceDocumentId(
system: DocumentPickerSystem,
id: string,
): boolean {
if (id === "." || id === "..") return false;
return (system === "PAPERLESS" ? PAPERLESS_ID : PAPRA_ID).test(id);
}
/** A connection as every route may show it: never the token. */
export interface DocumentSourceConnectionDto {
system: DocumentPickerSystem;
baseUrl: string;
organizationId: string | null;
hasToken: true;
lastVerifiedAt: string | null;
/** False once the operator removed the origin from the list. */
originAllowed: boolean;
}
export interface DocumentSourcesStatusDto {
/** The operator listed at least one origin; otherwise the picker is off. */
available: boolean;
/** The listed origins, so the settings card can say what is allowed. */
allowedOrigins: string[];
connections: DocumentSourceConnectionDto[];
}
export interface DocumentSourceTagDto {
id: string;
name: string;
}
/** Whether HealthLog already holds a result, by its source key. */
export type DocumentPickerResultState = "new" | "imported" | "deleted";
export interface DocumentSourceResultDto {
sourceId: string;
title: string;
/** YYYY-MM-DD, the document's own date in the source, or null. */
date: string | null;
tags: string[];
sizeBytes: number | null;
state: DocumentPickerResultState;
/** The vault document when `state` is `imported`. */
documentId: string | null;
}
export interface DocumentSourceSearchDto {
results: DocumentSourceResultDto[];
page: number;
hasMore: boolean;
}
export type DocumentImportOutcome = "imported" | "duplicate" | "deleted";
export interface DocumentSourceImportDto {
outcome: DocumentImportOutcome;
/** The vault document; null only when a deleted one has been purged. */
documentId: string | null;
/** The document was linked to the record the picker was opened from. */
linked: boolean;
}
/** Record kinds a picked document can be linked to on import. */
export const DOCUMENT_PICKER_LINK_KINDS = [
"conditionEpisode",
"encounter",
"vaccination",
] as const;
export type DocumentPickerLinkKind =
(typeof DOCUMENT_PICKER_LINK_KINDS)[number];
/**
* Why a request to a source failed, as the `meta.errorCode` values the routes
* emit. The route helper keys its status and wording on the last segment.
*/
export const DOCUMENT_SOURCE_ERROR_CODES = [
"documents.sources.unavailable",
"documents.sources.notConnected",
"documents.sources.originNotAllowed",
"documents.sources.unreachable",
"documents.sources.redirected",
"documents.sources.authRefused",
"documents.sources.permissionMissing",
"documents.sources.versionTooOld",
"documents.sources.badResponse",
"documents.sources.notFound",
"documents.sources.linkTargetNotFound",
"documents.sources.rateLimited",
] as const;
type SourceErrorSuffix<T> = T extends `documents.sources.${infer S}`
? S
: never;
export type DocumentSourceErrorCode = SourceErrorSuffix<
(typeof DOCUMENT_SOURCE_ERROR_CODES)[number]
>;
export function documentSourceErrorCode(
code: DocumentSourceErrorCode,
): (typeof DOCUMENT_SOURCE_ERROR_CODES)[number] {
return `documents.sources.${code}`;
}
+2
View File
@@ -620,6 +620,8 @@ export const NOT_IN_BACKUP_MODELS: Readonly<Record<string, string>> = {
"A CSRF nonce for a handshake that is valid for minutes and is already over by the time any backup is read.",
McpOAuthConnection:
"A grant issued to a client that was registered against this instance.",
DocumentSourceConnection:
"A person's API token for their Paperless-ngx or Papra instance. A backup file is not a credential store, same reasoning as ApiToken; and restored on another host it would point at an address that host's operator never allowed in DOCUMENT_SOURCE_ORIGINS. The documents imported through it are carried with the vault; reconnecting takes one form.",
IntegrationStatus:
"A ledger of what this deployment's syncs did. Restoring it onto another host would assert a sync history that host never had.",
+5
View File
@@ -177,6 +177,8 @@ export interface DocumentBackupEntry {
*/
sourceSystem?: string | null;
sourceId?: string | null;
/** v1.39.3 — the source instance (origin); absent in older files. */
sourceInstance?: string | null;
/** Canonical DR only: the import's hold on automatic AI reading. */
aiReadDeferred?: boolean;
}
@@ -309,6 +311,7 @@ export async function buildRecordsBackupSection(
lastIndexOutcome: true,
sourceSystem: true,
sourceId: true,
sourceInstance: true,
aiReadDeferred: true,
createdAt: true,
updatedAt: true,
@@ -330,6 +333,7 @@ export async function buildRecordsBackupSection(
summaryEncrypted: true,
sourceSystem: true,
sourceId: true,
sourceInstance: true,
createdAt: true,
},
});
@@ -545,6 +549,7 @@ export async function buildRecordsBackupSection(
: null,
sourceSystem: d.sourceSystem,
sourceId: d.sourceId,
sourceInstance: d.sourceInstance,
createdAt: d.createdAt.toISOString(),
};
+9
View File
@@ -30,6 +30,7 @@
import { Buffer } from "node:buffer";
import { prisma, toJson } from "@/lib/db";
import { normaliseSourceInstance } from "@/lib/validations/inbound-documents";
import { auditLog } from "@/lib/auth/audit";
import {
isStoredBackupReadError,
@@ -2057,6 +2058,14 @@ export async function restoreBackup(
sourceId: document.sourceSystem
? (document.sourceId ?? null)
: null,
// A file written before v1.39.3 has no instance: the key then
// matches any instance of its system, as it did when stored.
sourceInstance:
document.sourceSystem && document.sourceId
? document.sourceInstance
? normaliseSourceInstance(document.sourceInstance)
: null
: null,
createdAt: new Date(document.createdAt!),
updatedAt: new Date(document.updatedAt!),
})),
+6 -1
View File
@@ -58,9 +58,12 @@ export async function purgeExpiredDocumentTombstones(
userId: true,
sourceSystem: true,
sourceId: true,
sourceInstance: true,
// Further keys the import answered with this document (same bytes).
// They cascade away with the row, so they are copied here first.
sourceAliases: { select: { sourceSystem: true, sourceId: true } },
sourceAliases: {
select: { sourceSystem: true, sourceId: true, sourceInstance: true },
},
},
});
const keys = keyed.flatMap((row) => [
@@ -70,6 +73,7 @@ export async function purgeExpiredDocumentTombstones(
userId: row.userId,
sourceSystem: row.sourceSystem,
sourceId: row.sourceId,
sourceInstance: row.sourceInstance,
},
]
: []),
@@ -77,6 +81,7 @@ export async function purgeExpiredDocumentTombstones(
userId: row.userId,
sourceSystem: alias.sourceSystem,
sourceId: alias.sourceId,
sourceInstance: alias.sourceInstance,
})),
]);
if (keys.length > 0) {
+16
View File
@@ -136,6 +136,22 @@ export const ERROR_CODE_CATALOGUE: Readonly<Record<string, readonly string[]>> =
"documents.inbound.uploadBusy",
"documents.inbound.uploadTimeout",
"documents.inbound.vaccinationNotFound",
"documents.sources.authRefused",
"documents.sources.badResponse",
"documents.sources.browserOnly",
"documents.sources.invalidAddress",
"documents.sources.linkTargetNotFound",
"documents.sources.notConnected",
"documents.sources.notFound",
"documents.sources.organizationRequired",
"documents.sources.originNotAllowed",
"documents.sources.permissionMissing",
"documents.sources.rateLimited",
"documents.sources.redirected",
"documents.sources.tokenRequired",
"documents.sources.unavailable",
"documents.sources.unreachable",
"documents.sources.versionTooOld",
],
encounter: [
"encounter.invalid",
+402
View File
@@ -0,0 +1,402 @@
/**
* OpenAPI route table for the document picker (`/api/documents/sources/*`,
* #1038): a person's connection to Paperless-ngx or Papra, searching it, and
* importing picked documents into the vault.
*
* Every operation is cookie-only (declared per operation as `cookieAuth`): a
* Bearer, even a `["*"]` one, is refused 403 `documents.sources.browserOnly`,
* and so is a request acting on another person's record. The native client does
* not use these routes. Request bodies reuse the runtime Zod schemas.
*/
import type { ZodOpenApiObject } from "zod-openapi";
import { z } from "zod/v4";
import {
DOCUMENT_PICKER_LINK_KINDS,
DOCUMENT_PICKER_MAX_PAGE,
DOCUMENT_PICKER_QUERY_MAX,
DOCUMENT_PICKER_SYSTEMS,
} from "@/lib/documents/sources/types";
import {
documentSourceImportSchema,
documentSourceSaveSchema,
} from "@/lib/validations/document-sources";
import { dataEnvelope, errorEnvelope, stdResponses } from "./shared";
const cookieOnly = [{ cookieAuth: [] }];
const systemParameter = {
name: "system",
in: "path" as const,
required: true,
description: "`paperless` (Paperless-ngx) or `papra` (Papra).",
schema: { type: "string" as const, enum: ["paperless", "papra"] },
};
const systemEnum = z.enum(DOCUMENT_PICKER_SYSTEMS);
const connectionSchema = z
.object({
system: systemEnum,
baseUrl: z.string(),
organizationId: z.string().nullable(),
hasToken: z.literal(true),
lastVerifiedAt: z.string().nullable(),
originAllowed: z.boolean(),
})
.meta({ id: "DocumentSourceConnection" });
const statusSchema = z
.object({
available: z.boolean(),
allowedOrigins: z.array(z.string()),
connections: z.array(connectionSchema),
})
.meta({ id: "DocumentSourcesStatus" });
const tagSchema = z
.object({ id: z.string(), name: z.string() })
.meta({ id: "DocumentSourceTag" });
const resultSchema = z
.object({
sourceId: z.string(),
title: z.string(),
date: z.string().nullable(),
tags: z.array(z.string()),
sizeBytes: z.number().int().nullable(),
state: z.enum(["new", "imported", "deleted"]),
documentId: z.string().nullable(),
})
.meta({ id: "DocumentSourceResult" });
const searchSchema = z
.object({
results: z.array(resultSchema),
page: z.number().int(),
hasMore: z.boolean(),
})
.meta({ id: "DocumentSourceSearch" });
const importResultSchema = z
.object({
outcome: z.enum(["imported", "duplicate", "deleted"]),
documentId: z.string().nullable(),
linked: z.boolean(),
})
.meta({ id: "DocumentSourceImport" });
const refusal = {
description:
"Refused before any request to the source: `documents.sources.browserOnly` for a Bearer caller, `sharing.not_permitted` while acting on another person's record (a delegate or guardian never uses the owner's archive), or `module.disabled` when the Documents module is off.",
content: { "application/json": { schema: errorEnvelope } },
};
const unavailable = {
description:
"`documents.sources.unavailable`: the operator has not listed any origin in `DOCUMENT_SOURCE_ORIGINS`, so the picker is off. `documents.sources.notConnected`: no connection is saved for this system. `documents.sources.notFound`: the source has no such document, or (Papra) no such organization, including one the key's user is not a member of (Papra 403 `user.not_in_organization`) and one Papra cannot parse (400).",
content: { "application/json": { schema: errorEnvelope } },
};
const upstream = {
description:
"The source could not answer usefully, with the reason in `meta.errorCode` and, when the source answered at all, its HTTP status in `meta.upstreamStatus` (its body is never passed on): `documents.sources.unreachable` (no connection, timeout, or a destination refused at dial time), `documents.sources.redirected` (the source answered with a redirect, which is never followed), `documents.sources.authRefused` (401/403 from the source), `documents.sources.permissionMissing` (Papra: the key cannot read tags; needs `tags:read`), `documents.sources.versionTooOld` (Paperless-ngx older than 2.16 / API version 9), `documents.sources.badResponse` (not the documented JSON, or larger than 1 MiB).",
content: { "application/json": { schema: errorEnvelope } },
};
const originRefusal = {
description:
"`documents.sources.originNotAllowed`: the base address's origin is not on the operator's list (checked on every request, so a connection saved earlier is cut off once its origin is removed), or another validation failure.",
content: { "application/json": { schema: errorEnvelope } },
};
export const documentSourcePaths: NonNullable<ZodOpenApiObject["paths"]> = {
"/api/documents/sources": {
get: {
tags: ["Documents"],
summary: "Document archives: availability and connections",
description:
"Whether the operator enabled the document picker (`available`, true when `DOCUMENT_SOURCE_ORIGINS` lists at least one origin), the listed origins, and the caller's own saved connections to Paperless-ngx or Papra. A connection never carries its token: `hasToken` is always true for a saved one, and no route returns the token in whole or in part. `originAllowed` is false once the operator removed the connection's origin from the list. No request leaves the server. Needs neither the Documents module nor the list: with the list unset, `available` is false and any connection still stored is listed (with `originAllowed: false`) so it can be removed. Cookie session only; not delegable.",
security: cookieOnly,
responses: {
"200": {
description: "The picker's state for the caller.",
content: {
"application/json": {
schema: dataEnvelope(
statusSchema,
"DocumentSourcesStatusEnvelope",
),
},
},
},
"403": refusal,
...stdResponses,
},
},
},
"/api/documents/sources/{system}": {
put: {
tags: ["Documents"],
summary: "Save a connection to a document archive",
description:
"Saves or replaces the caller's connection to Paperless-ngx (`paperless`: base address and API token) or Papra (`papra`: base address, organization id, API key with `documents:read` and `tags:read`). The base address may carry a path; its origin must be listed in `DOCUMENT_SOURCE_ORIGINS`. The connection is tested live before anything is stored. Leaving out `token` keeps the stored one, but only when the new address is on the same origin as the saved one: the first save and a move to another origin must carry it (422 `documents.sources.tokenRequired`), so a stored token is never sent to an address it was not saved for. A Papra organization id is letters, digits, `_` and `-`. The token is stored encrypted (AES-256-GCM) and never returned. Ten saves and tests a minute per person. Cookie session only; not delegable.",
security: cookieOnly,
parameters: [systemParameter],
requestBody: {
required: true,
content: { "application/json": { schema: documentSourceSaveSchema } },
},
responses: {
"200": {
description: "Saved and verified. The connection, without its token.",
content: {
"application/json": {
schema: dataEnvelope(
connectionSchema,
"DocumentSourceConnectionEnvelope",
),
},
},
},
"403": refusal,
"404": unavailable,
"422": {
description:
"`documents.sources.originNotAllowed`, `documents.sources.invalidAddress`, `documents.sources.organizationRequired` (Papra), `documents.sources.tokenRequired` (first save without a token), or body validation.",
content: { "application/json": { schema: errorEnvelope } },
},
"502": upstream,
"401": stdResponses["401"],
"429": stdResponses["429"],
},
},
delete: {
tags: ["Documents"],
summary: "Remove a connection to a document archive",
description:
"Deletes the caller's connection for this system, and its token with it. Works even after the operator removed the origin from the list, and with the Documents module switched off. `disconnected` is false when there was nothing to delete. Cookie session only; not delegable.",
security: cookieOnly,
parameters: [systemParameter],
responses: {
"200": {
description: "Removed, or there was nothing to remove.",
content: {
"application/json": {
schema: dataEnvelope(
z
.object({ disconnected: z.boolean() })
.meta({ id: "DocumentSourceDisconnect" }),
"DocumentSourceDisconnectEnvelope",
),
},
},
},
"403": refusal,
...stdResponses,
},
},
},
"/api/documents/sources/{system}/test": {
post: {
tags: ["Documents"],
summary: "Test a saved connection",
description:
"One request to the source with the stored token. Answers `ok` with the round-trip time and stamps `lastVerifiedAt`, or the reason in `meta.errorCode`. Shares the save's allowance of ten a minute. Cookie session only; not delegable.",
security: cookieOnly,
parameters: [systemParameter],
responses: {
"200": {
description: "The source answered.",
content: {
"application/json": {
schema: dataEnvelope(
z
.object({
ok: z.literal(true),
latencyMs: z.number().int(),
lastVerifiedAt: z.string(),
})
.meta({ id: "DocumentSourceTest" }),
"DocumentSourceTestEnvelope",
),
},
},
},
"403": refusal,
"404": unavailable,
"422": originRefusal,
"502": upstream,
"401": stdResponses["401"],
"429": stdResponses["429"],
},
},
},
"/api/documents/sources/{system}/tags": {
get: {
tags: ["Documents"],
summary: "Tags of a connected archive",
description:
"The source's tags for the picker's filter, the first 250 by name, in the source's own ids. Shares the search allowance of sixty a minute. Cookie session only; not delegable.",
security: cookieOnly,
parameters: [systemParameter],
responses: {
"200": {
description: "The tags.",
content: {
"application/json": {
schema: dataEnvelope(
z
.object({ tags: z.array(tagSchema) })
.meta({ id: "DocumentSourceTags" }),
"DocumentSourceTagsEnvelope",
),
},
},
},
"403": refusal,
"404": unavailable,
"422": originRefusal,
"502": upstream,
"401": stdResponses["401"],
"429": stdResponses["429"],
},
},
},
"/api/documents/sources/{system}/search": {
get: {
tags: ["Documents"],
summary: "Search a connected archive by name, tag and date",
description:
"Searches the source 25 documents at a time. `q` matches the document's name (Paperless-ngx title, Papra name); `tag` narrows to one of the source's tags by id; `from` / `to` to a date range, inclusive (Paperless-ngx filters on `created__gte` / `created__lte`; Papra's range is applied to each page after it arrives). When a filter applied after arrival empties a page, the route reads on, up to five source pages, and `page` in the answer is the last source page read; ask for `page + 1` next. Each result carries what the vault already holds under that source key: `new`, `imported` with the vault document's `documentId`, or `deleted` (deleted in HealthLog; importing it again stores nothing). Nothing is stored. Sixty searches a minute, shared with the tag read. Cookie session only; not delegable.",
security: cookieOnly,
parameters: [
systemParameter,
{
name: "q",
in: "query",
required: false,
schema: { type: "string", maxLength: DOCUMENT_PICKER_QUERY_MAX },
},
{
name: "tag",
in: "query",
required: false,
schema: { type: "string", maxLength: 64 },
},
{
name: "from",
in: "query",
required: false,
schema: { type: "string", format: "date" },
},
{
name: "to",
in: "query",
required: false,
schema: { type: "string", format: "date" },
},
{
name: "page",
in: "query",
required: false,
schema: {
type: "integer",
minimum: 1,
maximum: DOCUMENT_PICKER_MAX_PAGE,
default: 1,
},
},
],
responses: {
"200": {
description: "One page of results.",
content: {
"application/json": {
schema: dataEnvelope(
searchSchema,
"DocumentSourceSearchEnvelope",
),
},
},
},
"403": refusal,
"404": unavailable,
"422": originRefusal,
"502": upstream,
"401": stdResponses["401"],
"429": stdResponses["429"],
},
},
},
"/api/documents/sources/{system}/import": {
post: {
tags: ["Documents"],
summary: "Import one picked document",
description:
"`sourceId` must be an id the system issues (Paperless-ngx: digits; Papra: letters, digits, `_`, `-`), else 422 before anything is fetched. Fetches one document from the source and stores it in the vault through the same path as an upload, with `sourceSystem` / `sourceId` / `sourceInstance` (the connection's origin) provenance; the same id from another instance is another document. One import per key runs at a time; a concurrent second one waits and then answers `duplicate`. The key check counts toward the lookup allowance, the fetch toward the search allowance, and only a file that arrived toward the upload allowance. A source key the vault already holds is answered without a download: `duplicate` for a live document (same id), `deleted` for one the person deleted in HealthLog, which is never stored again. Bytes already in the vault under another key are `duplicate` too, and the key is remembered for that document. `link` files the document against the record the picker was opened from (a condition, a visit or a vaccination, which must be the caller's own and live; checked before anything is fetched), including a duplicate. `kind` sets the document type (default OTHER). Automatic AI reading follows the person's own setting. Counts against the person's upload allowance (60 an hour) and storage quota. 201 for a new document, 200 otherwise. Cookie session only; not delegable.",
security: cookieOnly,
parameters: [systemParameter],
requestBody: {
required: true,
content: {
"application/json": { schema: documentSourceImportSchema },
},
},
responses: {
"201": {
description: "Imported.",
content: {
"application/json": {
schema: dataEnvelope(
importResultSchema,
"DocumentSourceImportEnvelope",
),
},
},
},
"200": {
description:
"Already held (`duplicate`) or deleted in HealthLog (`deleted`).",
content: {
"application/json": {
schema: dataEnvelope(
importResultSchema,
"DocumentSourceImportHeldEnvelope",
),
},
},
},
"403": refusal,
"404": {
description: `${unavailable.description} \`documents.sources.linkTargetNotFound\`: the record in \`link\` is not the caller's own live ${DOCUMENT_PICKER_LINK_KINDS.join(" / ")}.`,
content: { "application/json": { schema: errorEnvelope } },
},
"409": {
description:
"`documents.inbound.sourceAliasLimit`: these bytes are already stored under too many source ids.",
content: { "application/json": { schema: errorEnvelope } },
},
"413": {
description:
"`documents.inbound.fileTooLarge` (with `maxFileBytes`) or `documents.inbound.quotaExceeded` (with `quotaBytes` and `usedBytes`).",
content: { "application/json": { schema: errorEnvelope } },
},
"415": {
description:
"`documents.inbound.fileType`: the file is not a type the vault accepts.",
content: { "application/json": { schema: errorEnvelope } },
},
"422": originRefusal,
"502": upstream,
"401": stdResponses["401"],
"429": {
description:
"`documents.inbound.rateLimited` (the person's hourly upload allowance, with the rate-limit headers) or `documents.inbound.uploadBusy` (too many uploads in progress; `Retry-After: 1`).",
content: { "application/json": { schema: errorEnvelope } },
},
},
},
},
};
+26 -1
View File
@@ -24,6 +24,7 @@ import {
inboundConfirmSchema,
inboundFactEditSchema,
DOCUMENT_SOURCE_ID_MAX,
DOCUMENT_SOURCE_INSTANCE_MAX,
DOCUMENT_SOURCE_SYSTEMS,
DOCUMENT_SUMMARY_STATES,
INBOUND_DOCUMENT_KINDS,
@@ -417,12 +418,20 @@ export const inboundDocumentPaths: NonNullable<ZodOpenApiObject["paths"]> = {
description:
"Import source key, second half. Both halves or neither; half a key is 422.",
},
{
name: "sourceInstance",
in: "query",
required: false,
schema: { type: "string", maxLength: DOCUMENT_SOURCE_INSTANCE_MAX },
description:
"Optional, with a key. The instance of `sourceSystem` the id belongs to, as its http(s) address; normalised to the origin (`scheme://host[:port]`). The same id in two instances is two documents. Optional: a key stored without an instance (every upload before v1.39.3) matches any instance of its system, and a key sent without one matches any stored instance. If also sent in the form it must agree.",
},
],
tags: ["Documents"],
summary: "Store a document (no extraction)",
description:
'STORE-ONLY upload. Stores the raw document ENCRYPTED at rest with `status: STORED` and runs NO extraction — provider-free, no AI consent / budget / egress. A file is always filable, even with no document-scan provider configured. `multipart/form-data`: a `file` plus optional `title`, `kind`, `documentDate` (YYYY-MM-DD), and repeated `episodeIds` form fields (pre-link to the caller\'s illness/condition episodes). Accepted types (magic-byte sniffed, never the wire Content-Type): PDF/JPEG/PNG/WebP/GIF render inline; Office (docx/xlsx/pptx/doc/xls/ppt), text/CSV/Markdown/RTF, TIFF, HEIC/HEIF, XML/JSON are stored verbatim and served download-only. HEIC is stored as-is but attachment-only — prefer transcoding to JPEG client-side for inline preview parity. Error contract: `413` with `meta.reason = "fileTooLarge"` (+ `maxFileBytes`) or `"quotaExceeded"` (+ `quotaBytes`, `usedBytes`); `415` with `meta.reason = "unsupportedType"`. A same-user duplicate (same bytes) returns 200 + `meta.duplicate: true` with the existing row — not an error. `Idempotency-Key` honoured. Read `GET /api/documents/inbound/usage` for the effective limits before offering an upload. AI extraction is a separate opt-in action — see `POST /api/documents/inbound/{id}/extract`.\n\n' +
"**Importing from another system (v1.39.2).** Optional `sourceSystem` (`PAPERLESS`, `PAPRA`, `OTHER`) and `sourceId` (its id there, printable, up to 128 characters; needs `sourceSystem`) key the upload. A re-send with a known key is a duplicate: 200 with the live row (`meta.duplicate: true`), or, when the owner DELETED that document, 200 with `data.deleted: true` and `meta.deleted: true` and nothing stored; this holds after the 30-day purge too. The key may also ride the query string (`?sourceSystem=…&sourceId=…`): a key the vault already holds is then answered at once, the file is not stored or processed, and the request counts only toward the lookup allowance shared with `GET /api/documents/inbound/source` (5000 an hour per token, per person for a session; 429 past it), not toward the hourly upload limit. A key in both places must agree (else 422). Every other upload counts toward the hourly upload limit, duplicates included. A key that is answered with an existing document by its bytes is remembered for that document, so deleting it keeps that key deleted too; one document takes at most 20 such keys, beyond which the upload is refused 409 (`documents.inbound.sourceAliasLimit`). `OTHER` is one shared namespace per account: two different systems both sent as `OTHER` must not reuse each other\'s ids. `aiRead=defer` holds back automatic AI reading for this upload: the thumbnail and a local text index still run, the summary and lab staging do not, and the document stays out of every automatic AI path (the summary catch-up, the automatic index, Index all for search) until the person reads it with AI or generates its summary. Absent, behaviour is unchanged.\n\n" +
"**Importing from another system (v1.39.2).** Optional `sourceSystem` (`PAPERLESS`, `PAPRA`, `OTHER`) and `sourceId` (its id there, printable, up to 128 characters, never `.` or `..`; needs `sourceSystem`) key the upload, optionally with `sourceInstance` (v1.39.3: the address of the Paperless-ngx or Papra it came from, normalised to its origin; without it the key matches any instance of the system). A re-send with a known key is a duplicate: 200 with the live row (`meta.duplicate: true`), or, when the owner DELETED that document, 200 with `data.deleted: true` and `meta.deleted: true` and nothing stored; this holds after the 30-day purge too. The key may also ride the query string (`?sourceSystem=…&sourceId=…`): a key the vault already holds is then answered at once, the file is not stored or processed, and the request counts only toward the lookup allowance shared with `GET /api/documents/inbound/source` (5000 an hour per token, per person for a session; 429 past it), not toward the hourly upload limit. A key in both places must agree (else 422). Every other upload counts toward the hourly upload limit, duplicates included. A key that is answered with an existing document by its bytes is remembered for that document, so deleting it keeps that key deleted too; one document takes at most 20 such keys, beyond which the upload is refused 409 (`documents.inbound.sourceAliasLimit`). `OTHER` is one shared namespace per account: two different systems both sent as `OTHER` must not reuse each other\'s ids. `aiRead=defer` holds back automatic AI reading for this upload: the thumbnail and a local text index still run, the summary and lab staging do not, and the document stays out of every automatic AI path (the summary catch-up, the automatic index, Index all for search) until the person reads it with AI or generates its summary. Absent, behaviour is unchanged.\n\n" +
"**Narrow token.** Also reachable with a `documents:write` Bearer (minted at `POST /api/tokens/documents`), which reaches only this route and the source-key lookup `GET /api/documents/inbound/source`. Such a caller draws on a bucket of its own, keyed on the token (default 120 an hour, `DOCUMENT_UPLOAD_LIMIT_PER_HOUR`, clamped 1-1000; the 429 carries `Retry-After` and `X-RateLimit-*`), and gets a `DocumentUploadReceipt` instead of the stored row; its 413 `quotaExceeded` carries no `quotaBytes` / `usedBytes`. The response is an untagged union (row or receipt) on purpose: a discriminator would need a new field on the row every existing client decodes. A cookie or wildcard caller keeps the 60-an-hour per-user bucket and the full row. With the vault module off the answer is 403 `module.disabled` for every caller.",
requestBody: {
required: true,
@@ -461,6 +470,14 @@ export const inboundDocumentPaths: NonNullable<ZodOpenApiObject["paths"]> = {
description:
"Optional. The document's id in `sourceSystem`; requires it. Unique per account across live and deleted documents.",
}),
sourceInstance: z
.string()
.max(DOCUMENT_SOURCE_INSTANCE_MAX)
.optional()
.meta({
description:
"Optional; requires `sourceId`. The instance of `sourceSystem` the id belongs to, as its http(s) address; normalised to the origin (`scheme://host[:port]`). The same id in two instances is two documents. Optional: a key stored without an instance (every upload before v1.39.3) matches any instance of its system, and a key sent without one matches any stored instance.",
}),
aiRead: z.enum(["defer"]).optional().meta({
description:
"Optional. `defer` holds back automatic AI reading for this upload (local index and thumbnail only).",
@@ -527,6 +544,14 @@ export const inboundDocumentPaths: NonNullable<ZodOpenApiObject["paths"]> = {
required: true,
schema: { type: "string", maxLength: DOCUMENT_SOURCE_ID_MAX },
},
{
name: "sourceInstance",
in: "query",
required: false,
schema: { type: "string", maxLength: DOCUMENT_SOURCE_INSTANCE_MAX },
description:
"The instance of `sourceSystem` the id belongs to, as its http(s) address; normalised to the origin (`scheme://host[:port]`). The same id in two instances is two documents. Optional: a key stored without an instance (every upload before v1.39.3) matches any instance of its system, and a key sent without one matches any stored instance.",
},
],
responses: {
"200": {
+5
View File
@@ -50,6 +50,7 @@ import { cyclePaths } from "./cycle";
import { dashboardWidgetPaths } from "./dashboard";
import { biomarkerPaths } from "./biomarkers";
import { inboundDocumentPaths } from "./documents";
import { documentSourcePaths } from "./document-sources";
import { devicePaths } from "./devices";
import { exportPaths } from "./export";
import { familyHistoryPaths } from "./family-history";
@@ -170,6 +171,10 @@ export const openApiPaths: NonNullable<ZodOpenApiObject["paths"]> = {
// discover: a copy this instance can no longer decrypt (appended, spread
// order is load-bearing).
...adminBackupPaths,
// v1.39.3 — the document picker: a person's connection to Paperless-ngx or
// Papra, searching it and importing picked documents; cookie-only
// (appended, spread order is load-bearing).
...documentSourcePaths,
// The environmental-context overview. Its own module because nothing else
// owns the surface (appended, spread order is load-bearing).
//
+23
View File
@@ -81,4 +81,27 @@ export const documentKeys = {
* toggle; a flip invalidates the document AI capability probe.
*/
documentsAutoAiRead: () => ["documents", "auto-ai-read"] as const,
/**
* The document picker (#1038): its own top-level prefix, deliberately NOT
* under `["documents"]`. Every vault write invalidates that prefix, and a
* search or tag read here is a request to the person's Paperless-ngx or
* Papra instance; a filing action in the vault must not re-send them.
*/
documentSources: () => ["documentSources"] as const,
/** Availability + the person's connections (`GET /api/documents/sources`). */
documentSourcesStatus: () => ["documentSources", "status"] as const,
/** One source's tags for the filter. */
documentSourceTags: (system: string) =>
["documentSources", "tags", system] as const,
/** One page of one source's search. */
documentSourceSearch: (system: string, search: DocumentSourceSearchKey) =>
["documentSources", "search", system, search] as const,
};
/** The facets that make one picker search distinct from another. */
export interface DocumentSourceSearchKey {
q: string;
tag?: string;
from?: string;
to?: string;
}
+9
View File
@@ -83,6 +83,7 @@ import {
} from "@/lib/validations/emergency-profile";
import {
DOCUMENT_SOURCE_ID_MAX,
DOCUMENT_SOURCE_INSTANCE_MAX,
DOCUMENT_SOURCE_SYSTEMS,
} from "@/lib/validations/inbound-documents";
import { REMINDER_EVENT_SOURCES } from "@/lib/measurement-reminders/satisfy";
@@ -1636,6 +1637,14 @@ const documentBackupSchema = z
// restore cannot write a system the detail sheet has no name for.
sourceSystem: z.enum(DOCUMENT_SOURCE_SYSTEMS).nullable().optional(),
sourceId: z.string().max(DOCUMENT_SOURCE_ID_MAX).nullable().optional(),
// v1.39.3 — the source instance (an origin). Length-bounded here and
// normalised on restore; a value that is not an origin restores as null
// (matches any instance) rather than failing the restore.
sourceInstance: z
.string()
.max(DOCUMENT_SOURCE_INSTANCE_MAX)
.nullable()
.optional(),
aiReadDeferred: z.boolean().optional(),
summary: z.string().nullable().optional(),
createdAt: isoDateTime.optional(),
+90
View File
@@ -0,0 +1,90 @@
/**
* Request schemas of the document picker (#1038): saving a connection,
* searching a source, importing one picked document. No `userId` field in any
* of them; the owner is always the authenticated session.
*/
import { z } from "zod/v4";
import { DOCUMENT_SOURCE_BASE_URL_MAX } from "@/lib/documents/sources/origins";
import {
DOCUMENT_PICKER_LINK_KINDS,
DOCUMENT_PICKER_MAX_PAGE,
DOCUMENT_PICKER_QUERY_MAX,
PAPRA_ORG_ID,
} from "@/lib/documents/sources/types";
import {
documentSourceIdSchema,
INBOUND_DOCUMENT_KINDS,
} from "@/lib/validations/inbound-documents";
/** Printable ASCII without spaces: an id or a key, never prose. */
const printable = /^[\x21-\x7e]+$/u;
const isoDate = z
.string()
.regex(/^\d{4}-\d{2}-\d{2}$/u, "Expected a YYYY-MM-DD date");
/** Longest API token or key a connection stores. */
export const DOCUMENT_SOURCE_TOKEN_MAX = 512;
/**
* Save a connection. `token` may be left out when a connection for the system
* already exists: the stored one is kept, so the address or the organization
* can change without typing the key again. The first save must carry it (the
* route says so). `organizationId` is Papra's and required there.
*/
export const documentSourceSaveSchema = z.object({
baseUrl: z.string().trim().min(1).max(DOCUMENT_SOURCE_BASE_URL_MAX),
organizationId: z
.string()
.trim()
.regex(PAPRA_ORG_ID, "Expected a Papra organization id (org_…)")
.optional(),
token: z
.string()
.trim()
.min(1)
.max(DOCUMENT_SOURCE_TOKEN_MAX)
.regex(printable, "Expected printable characters without spaces")
.optional(),
});
export type DocumentSourceSaveInput = z.infer<typeof documentSourceSaveSchema>;
export const documentSourceSearchSchema = z
.object({
q: z.string().trim().max(DOCUMENT_PICKER_QUERY_MAX).default(""),
tag: z
.string()
.trim()
.min(1)
.max(64)
.regex(printable, "Expected printable characters without spaces")
.optional(),
from: isoDate.optional(),
to: isoDate.optional(),
page: z.coerce
.number()
.int()
.min(1)
.max(DOCUMENT_PICKER_MAX_PAGE)
.default(1),
})
.refine((v) => !v.from || !v.to || v.from <= v.to, {
path: ["to"],
message: "The end date is before the start date",
});
export const documentSourceImportSchema = z.object({
sourceId: documentSourceIdSchema,
kind: z.enum(INBOUND_DOCUMENT_KINDS).optional(),
/** The record the picker was opened from, to link the document to. */
link: z
.object({
kind: z.enum(DOCUMENT_PICKER_LINK_KINDS),
id: z.string().trim().min(1).max(64),
})
.optional(),
});
export type DocumentSourceImportInput = z.infer<
typeof documentSourceImportSchema
>;
+47 -2
View File
@@ -701,12 +701,46 @@ export const DOCUMENT_SOURCE_ID_MAX = 128;
* The document's id in its source system. Printable ASCII only: it is a key,
* not prose, and it is echoed on the detail sheet.
*/
const documentSourceIdSchema = z
export const documentSourceIdSchema = z
.string()
.trim()
.min(1)
.max(DOCUMENT_SOURCE_ID_MAX)
.regex(/^[\x21-\x7e]+$/u, "Expected printable characters without spaces");
.regex(/^[\x21-\x7e]+$/u, "Expected printable characters without spaces")
// A path segment of its own name: an importer that builds a URL from the id
// must never be handed one that walks up the source's path.
.refine((v) => v !== "." && v !== "..", "Not a document id");
/** Longest source instance (an origin) a key may carry. */
export const DOCUMENT_SOURCE_INSTANCE_MAX = 512;
/**
* v1.39.3 — the instance of the source system a key belongs to, as the
* normalised origin (`scheme://host[:port]`) of the Paperless-ngx or Papra it
* came from, or null when the value is not one. A path, query or credentials
* in the value are dropped: two addresses on one origin are one instance.
*/
export function normaliseSourceInstance(value: string): string | null {
try {
const url = new URL(value.trim());
if (url.protocol !== "http:" && url.protocol !== "https:") return null;
if (url.username || url.password) return null;
return url.origin;
} catch {
return null;
}
}
/** The instance as a request field: validated, then normalised to its origin. */
export const documentSourceInstanceSchema = z
.string()
.trim()
.min(1)
.max(DOCUMENT_SOURCE_INSTANCE_MAX)
.refine((v) => normaliseSourceInstance(v) !== null, {
message: "Expected the http(s) address of the source instance",
})
.transform((v) => normaliseSourceInstance(v) as string);
/**
* A source key on its own — the query-string form the upload accepts ahead of
@@ -715,6 +749,8 @@ const documentSourceIdSchema = z
export const documentSourceKeySchema = z.object({
sourceSystem: z.enum(DOCUMENT_SOURCE_SYSTEMS),
sourceId: documentSourceIdSchema,
/** Optional: without it the key matches any instance of the system. */
sourceInstance: documentSourceInstanceSchema.optional(),
});
export const documentCreateSchema = z
@@ -731,6 +767,11 @@ export const documentCreateSchema = z
* not prose, and it is echoed on the detail sheet.
*/
sourceId: documentSourceIdSchema.optional(),
/**
* v1.39.3 — which instance of `sourceSystem` (its origin). Optional; a
* key without one matches any instance.
*/
sourceInstance: documentSourceInstanceSchema.optional(),
/**
* `defer` holds back automatic AI reading for this upload: the thumbnail
* and a local text index still run, the summary and the lab staging do
@@ -741,6 +782,10 @@ export const documentCreateSchema = z
.refine((v) => v.sourceId === undefined || v.sourceSystem !== undefined, {
path: ["sourceSystem"],
message: "sourceId needs a sourceSystem",
})
.refine((v) => v.sourceInstance === undefined || v.sourceId !== undefined, {
path: ["sourceInstance"],
message: "sourceInstance needs a sourceId",
});
export type DocumentCreateInput = z.infer<typeof documentCreateSchema>;