tom/HealthLog
1
0 Fork 0
mirror of https://github.com/MBombeck/HealthLog.git synced 2026-11-12 06:06:10 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(documents): key imports by source instance, keep a token on its origin, and harden the import path

Browse Source 
- Source keys carry the normalised origin of the Paperless-ngx or Papra
  they came from (source_instance on the document, the alias and the
  purge ledger; unique NULLS NOT DISTINCT). A key stored without one
  (v1.39.2 imports) matches any instance of its system, and a lookup
  without one matches any stored instance. The upload route, the lookup
  route and the import script accept or send it; backups carry it.
- Saving a connection reuses the stored token only for the same origin;
  a new origin needs the token again (422 tokenRequired).
- Document ids are checked against what each system issues (Paperless
  digits, Papra [A-Za-z0-9_-], never . or ..) before any URL is built;
  the Papra organization id likewise.
- Paperless API 9 filters dates on created__gte / created__lte, in the
  picker and the script. A search page the local re-check empties reads
  on, up to five source pages. Tags are read past the first 250.
- The import's key pre-check uses the lookup allowance, a failed fetch
  no longer charges the upload allowance, and one import per key runs at
  a time under an advisory lock, so a double click downloads once.
- Reading and deleting one's own connection works with the documents
  module off and the operator list unset.
This commit is contained in:
Marc-André Bombeck committed 2026-09-26 14:20:17 +02:00
1 parent 6029c7c30d
commit 1eac05b454
24 files changed
+886 -209
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

src/app/api/documents/inbound/__tests__/route.test.ts
+8 -6
View File
@@ -51,10 +51,10 @@ vi.mock("@/lib/db", () => {
findFirst: vi.fn(),
},
documentImportKey: {
findUnique: vi.fn(),
findFirst: vi.fn(),
},
documentSourceAlias: {
findUnique: vi.fn(),
findFirst: vi.fn(),
createMany: vi.fn(),
count: vi.fn(),
},
@@ -239,10 +239,10 @@ beforeEach(() => {
[] as never,
);
vi.mocked(prisma.inboundDocument.findFirst).mockResolvedValue(null as never);
vi.mocked(prisma.documentImportKey.findUnique).mockResolvedValue(
vi.mocked(prisma.documentImportKey.findFirst).mockResolvedValue(
null as never,
);
vi.mocked(prisma.documentSourceAlias.findUnique).mockResolvedValue(
vi.mocked(prisma.documentSourceAlias.findFirst).mockResolvedValue(
null as never,
);
vi.mocked(prisma.documentSourceAlias.createMany).mockResolvedValue({
@@ -783,7 +783,7 @@ describe("POST /api/documents/inbound — document import", () => {
});
it("remembers a purged import through the ledger", async () => {
vi.mocked(prisma.documentImportKey.findUnique).mockResolvedValue({
vi.mocked(prisma.documentImportKey.findFirst).mockResolvedValue({
id: "key-1",
} as never);
const res = await post(
@@ -909,6 +909,8 @@ describe("POST /api/documents/inbound — document import", () => {
userId: "user-1",
documentId: "doc-manual",
sourceSystem: "PAPRA",
// Sent without an instance: the alias matches any Papra.
sourceInstance: null,
sourceId: "doc_9",
},
],
@@ -936,7 +938,7 @@ describe("POST /api/documents/inbound — document import", () => {
});
it("resolves a key held as an alias of a deleted document to deleted", async () => {
vi.mocked(prisma.documentSourceAlias.findUnique).mockResolvedValue({
vi.mocked(prisma.documentSourceAlias.findFirst).mockResolvedValue({
document: docRow({ id: "doc-gone", deletedAt: new Date() }),
} as never);
const res = await post(
src/app/api/documents/inbound/route.ts
+20 -2
View File
@@ -274,14 +274,17 @@ async function processUpload(
const url = new URL(request.url);
const querySystem = url.searchParams.get("sourceSystem");
const queryId = url.searchParams.get("sourceId");
const queryInstance = url.searchParams.get("sourceInstance");
let queryKey: {
sourceSystem: DocumentSourceSystemValue;
sourceId: string;
sourceInstance?: string;
} | null = null;
if (querySystem !== null || queryId !== null) {
const parsedKey = documentSourceKeySchema.safeParse({
sourceSystem: querySystem ?? undefined,
sourceId: queryId ?? undefined,
sourceInstance: queryInstance ?? undefined,
});
if (!parsedKey.success) {
return apiValidationError(
@@ -312,6 +315,7 @@ async function processUpload(
user.id,
queryKey.sourceSystem,
queryKey.sourceId,
queryKey.sourceInstance ?? null,
);
if (match) {
return answerSourceKey(
@@ -408,6 +412,7 @@ async function processUpload(
encounterIds: rawEncounterIds.length > 0 ? rawEncounterIds : undefined,
sourceSystem: formData.get("sourceSystem") ?? undefined,
sourceId: formData.get("sourceId") ?? undefined,
sourceInstance: formData.get("sourceInstance") ?? undefined,
aiRead: formData.get("aiRead") ?? undefined,
});
if (!parsed.success) {
@@ -425,10 +430,14 @@ async function processUpload(
// both they have to agree.
const formSystem = parsed.data.sourceSystem ?? null;
const formId = formSystem ? (parsed.data.sourceId ?? null) : null;
const formInstance = formId ? (parsed.data.sourceInstance ?? null) : null;
if (
queryKey &&
formSystem !== null &&
(formSystem !== queryKey.sourceSystem || formId !== queryKey.sourceId)
(formSystem !== queryKey.sourceSystem ||
formId !== queryKey.sourceId ||
(formInstance !== null &&
formInstance !== (queryKey.sourceInstance ?? null)))
) {
return apiError(
"The source key in the address and in the form differ.",
@@ -438,6 +447,9 @@ async function processUpload(
}
const sourceSystem = queryKey?.sourceSystem ?? formSystem;
const sourceId = queryKey?.sourceId ?? formId;
const sourceInstance = sourceId
? (queryKey?.sourceInstance ?? formInstance ?? null)
: null;
const ctx: UploadContext = {
userId: user.id,
scoped,
@@ -451,7 +463,12 @@ async function processUpload(
// count (the unique index has no `deleted_at` predicate), and past the purge
// the ledger remembers.
if (sourceSystem && sourceId && !queryKey) {
const match = await findSourceKey(user.id, sourceSystem, sourceId);
const match = await findSourceKey(
user.id,
sourceSystem,
sourceId,
sourceInstance,
);
if (match) return answerSourceKey(ctx, match);
}
@@ -478,6 +495,7 @@ async function processUpload(
encounterIds: parsed.data.encounterIds ?? [],
sourceSystem,
sourceId,
sourceInstance,
aiDeferred: ctx.aiDeferred,
sourceKeyChecked: true,
limits,
src/app/api/documents/inbound/source/route.ts
+2
View File
@@ -57,6 +57,7 @@ export const GET = apiHandler(async (request: Request) => {
const parsed = documentSourceKeySchema.safeParse({
sourceSystem: url.searchParams.get("sourceSystem") ?? undefined,
sourceId: url.searchParams.get("sourceId") ?? undefined,
sourceInstance: url.searchParams.get("sourceInstance") ?? undefined,
});
if (!parsed.success) {
return apiValidationError(
@@ -71,6 +72,7 @@ export const GET = apiHandler(async (request: Request) => {
user.id,
parsed.data.sourceSystem,
parsed.data.sourceId,
parsed.data.sourceInstance ?? null,
);
annotate({
action: { name: "documents.vault.sourceLookup" },
src/app/api/documents/sources/[system]/import/route.ts
+185 -98
View File
@@ -12,13 +12,16 @@
* imported (or already holding this key as an alias) is a `duplicate`,
* and one the person deleted here is `deleted` — nothing is fetched and
* nothing is stored, exactly as the token upload answers the same key;
* 3. only then the vault's own upload allowance (60 an hour, shared with
* uploads from the web and the phone) and an upload slot are taken, the
* document's details and its original file are fetched (bounded by the
* vault's per-file cap), and the file goes through `ingestDocument`, the
* same function the upload route stores through: type check, content
* dedup, quota, provenance (`sourceSystem` / `sourceId`), the background
* jobs;
* 3. only then is the source asked (on the search allowance, like every
* request to it) and an upload slot taken. One import per key and person
* runs at a time under an advisory lock, so a double click downloads the
* document once. The details and the original file are fetched (bounded
* by the vault's per-file cap); only a file that arrived is charged to
* the vault's own upload allowance (60 an hour, shared with uploads from
* the web and the phone), and it goes through `ingestDocument`, the same
* function the upload route stores through: type check, content dedup,
* quota, provenance (`sourceSystem` / `sourceId` / `sourceInstance`), the
* background jobs;
* 4. the document is linked to the record through `src/lib/links/`.
*
* AI reading follows the person's own setting, unlike the token upload's
@@ -48,18 +51,26 @@ import {
UPLOAD_LIMIT_PER_HOUR,
UPLOAD_WINDOW_MS,
} from "@/lib/documents/ingest";
import { findSourceKey, type SourceKeyMatch } from "@/lib/documents/source-key";
import {
checkSourceLookupRateLimit,
findSourceKey,
type SourceKeyMatch,
} from "@/lib/documents/source-key";
import type { SourceClient } from "@/lib/documents/sources/clients";
import { clientFor, loadConnection } from "@/lib/documents/sources/connections";
import {
admitDocumentSourceCaller,
checkSearchRateLimit,
responseForSourceFailure,
sourceErrorResponse,
systemParam,
} from "@/lib/documents/sources/route-support";
import type {
DocumentImportOutcome,
DocumentPickerLinkKind,
DocumentSourceImportDto,
import {
isSourceDocumentId,
type DocumentImportOutcome,
type DocumentPickerLinkKind,
type DocumentPickerSystem,
type DocumentSourceImportDto,
} from "@/lib/documents/sources/types";
import {
acquireDocumentUploadSlot,
@@ -69,11 +80,17 @@ import { linkTargets } from "@/lib/links";
import { annotate } from "@/lib/logging/context";
import { checkRateLimit, rateLimitHeaders } from "@/lib/rate-limit";
import { documentSourceImportSchema } from "@/lib/validations/document-sources";
import type { InboundDocumentKindValue } from "@/lib/validations/inbound-documents";
export const dynamic = "force-dynamic";
type RouteParams = { params: Promise<{ system: string }> };
/** How long an import waits for a database connection to hold its lock on. */
const IMPORT_LOCK_WAIT_MS = 10_000;
/** Longest a single import may hold its key: the download plus the store. */
const IMPORT_LOCK_HOLD_MS = 150_000;
interface LinkTarget {
kind: DocumentPickerLinkKind;
id: string;
@@ -172,6 +189,10 @@ export const POST = apiHandler(
);
}
const { sourceId, kind, link } = parsed.data;
// Only an id this system could have issued goes into a request path.
if (!isSourceDocumentId(system, sourceId)) {
return apiError("Not a document id of this archive.", 422);
}
const row = await loadConnection(userId, system);
if (!row) return sourceErrorResponse("notConnected");
@@ -181,25 +202,38 @@ export const POST = apiHandler(
} catch (err) {
return responseForSourceFailure(err);
}
const instance = client.instance;
if (link && !(await ownsLinkTarget(userId, link))) {
return sourceErrorResponse("linkTargetNotFound");
}
// The source key answers before anything is downloaded or charged.
const held = await findSourceKey(userId, system, sourceId);
// The source key answers before anything is downloaded or charged, on the
// same metered lookup allowance the upload's key check draws on.
const lookupRl = await checkSourceLookupRateLimit(
false,
auth.session.id,
userId,
);
if (!lookupRl.allowed) {
return sourceErrorResponse(
"rateLimited",
undefined,
rateLimitHeaders(lookupRl),
);
}
const held = await findSourceKey(userId, system, sourceId, instance);
if (held) return answerHeldKey(system, userId, link, held);
const rl = await checkRateLimit(
personalUploadBucket(userId),
UPLOAD_LIMIT_PER_HOUR,
UPLOAD_WINDOW_MS,
);
if (!rl.allowed) {
return apiError("Too many uploads. Try again later.", 429, {
errorCode: "documents.inbound.rateLimited",
headers: rateLimitHeaders(rl),
});
// A request to the source counts against the search allowance like any
// other; the upload allowance is charged only once a file arrived.
const fetchRl = await checkSearchRateLimit(userId);
if (!fetchRl.allowed) {
return sourceErrorResponse(
"rateLimited",
undefined,
rateLimitHeaders(fetchRl),
);
}
const release = acquireDocumentUploadSlot(userId);
@@ -216,83 +250,136 @@ export const POST = apiHandler(
}
try {
const limits = await resolveDocumentLimits(userId);
let meta;
let file;
try {
meta = await client.document(sourceId);
file = await client.download(sourceId, limits.maxFileBytes);
} catch (err) {
return responseForSourceFailure(err);
}
const result = await ingestDocument({
userId,
scoped: false,
ipAddress: getClientIp(request),
bytes: Buffer.from(file.bytes),
filename: meta.filename ?? file.filename,
title: meta.title,
kind: kind ?? null,
documentDate: meta.date,
episodeIds: [],
encounterIds: [],
sourceSystem: system,
sourceId,
aiDeferred: false,
sourceKeyChecked: true,
limits,
// The import is data; only the summary queued after a fresh insert
// follows the capability (and the person's auto-read setting).
documentAi: () => getAiCapability("documentAi"),
});
switch (result.kind) {
case "sourceKey":
return answerHeldKey(system, userId, link, result.match);
case "duplicate": {
// The same bytes were already in the vault (an upload, or another
// archive); the key is now remembered for that document.
const linked = await linkDocument(userId, link, result.document.id);
return respond(system, "duplicate", result.document.id, linked);
}
case "stored": {
const linked = await linkDocument(userId, link, result.document.id);
return respond(system, "imported", result.document.id, linked);
}
case "tooLarge":
return apiError("File is too large.", 413, {
errorCode: "documents.inbound.fileTooLarge",
reason: "fileTooLarge",
maxFileBytes: result.maxFileBytes,
// One import of one key at a time per person: a second tab or a double
// click waits here, then finds the key the first one stored, so the
// document is downloaded once. The lock is transaction-scoped and the
// transaction holds nothing else; the store runs in its own.
return await prisma.$transaction(
async (tx) => {
await tx.$queryRaw`
SELECT 1 AS locked
FROM pg_advisory_xact_lock(hashtextextended(${`documents-import:${userId}:${system}:${instance}:${sourceId}`}, 0))
`;
const raced = await findSourceKey(userId, system, sourceId, instance);
if (raced) return answerHeldKey(system, userId, link, raced);
return importOne({
request,
client,
userId,
system,
instance,
sourceId,
kind: kind ?? null,
link,
});
case "empty":
case "unsupportedType":
return apiError("This file type is not supported.", 415, {
errorCode: "documents.inbound.fileType",
reason: "unsupportedType",
});
case "quotaExceeded":
return apiError("Storage quota exceeded.", 413, {
errorCode: "documents.inbound.quotaExceeded",
reason: "quotaExceeded",
quotaBytes: result.quotaBytes,
usedBytes: result.usedBytes,
});
case "aliasLimit":
return apiError(
"This file is already stored under too many source ids.",
409,
{ errorCode: "documents.inbound.sourceAliasLimit" },
);
case "episodeNotFound":
case "encounterNotFound":
// No pre-links are passed on this path; unreachable, but a result
// the switch does not name must not fall through silently.
return sourceErrorResponse("linkTargetNotFound");
}
},
{ maxWait: IMPORT_LOCK_WAIT_MS, timeout: IMPORT_LOCK_HOLD_MS },
);
} finally {
release();
}
},
);
/** Fetch one document and store it; the caller holds the key's lock. */
async function importOne(args: {
request: Request;
client: SourceClient;
userId: string;
system: DocumentPickerSystem;
instance: string;
sourceId: string;
kind: InboundDocumentKindValue | null;
link: LinkTarget | undefined;
}): Promise<Response> {
const { request, client, userId, system, instance, sourceId, kind, link } =
args;
const limits = await resolveDocumentLimits(userId);
let meta;
let file;
try {
meta = await client.document(sourceId);
file = await client.download(sourceId, limits.maxFileBytes);
} catch (err) {
return responseForSourceFailure(err);
}
const rl = await checkRateLimit(
personalUploadBucket(userId),
UPLOAD_LIMIT_PER_HOUR,
UPLOAD_WINDOW_MS,
);
if (!rl.allowed) {
return apiError("Too many uploads. Try again later.", 429, {
errorCode: "documents.inbound.rateLimited",
headers: rateLimitHeaders(rl),
});
}
const result = await ingestDocument({
userId,
scoped: false,
ipAddress: getClientIp(request),
bytes: Buffer.from(file.bytes),
filename: meta.filename ?? file.filename,
title: meta.title,
kind,
documentDate: meta.date,
episodeIds: [],
encounterIds: [],
sourceSystem: system,
sourceId,
sourceInstance: instance,
aiDeferred: false,
sourceKeyChecked: true,
limits,
// The import is data; only the summary queued after a fresh insert
// follows the capability (and the person's auto-read setting).
documentAi: () => getAiCapability("documentAi"),
});
switch (result.kind) {
case "sourceKey":
return answerHeldKey(system, userId, link, result.match);
case "duplicate": {
// The same bytes were already in the vault (an upload, or another
// archive); the key is now remembered for that document.
const linked = await linkDocument(userId, link, result.document.id);
return respond(system, "duplicate", result.document.id, linked);
}
case "stored": {
const linked = await linkDocument(userId, link, result.document.id);
return respond(system, "imported", result.document.id, linked);
}
case "tooLarge":
return apiError("File is too large.", 413, {
errorCode: "documents.inbound.fileTooLarge",
reason: "fileTooLarge",
maxFileBytes: result.maxFileBytes,
});
case "empty":
case "unsupportedType":
return apiError("This file type is not supported.", 415, {
errorCode: "documents.inbound.fileType",
reason: "unsupportedType",
});
case "quotaExceeded":
return apiError("Storage quota exceeded.", 413, {
errorCode: "documents.inbound.quotaExceeded",
reason: "quotaExceeded",
quotaBytes: result.quotaBytes,
usedBytes: result.usedBytes,
});
case "aliasLimit":
return apiError(
"This file is already stored under too many source ids.",
409,
{ errorCode: "documents.inbound.sourceAliasLimit" },
);
case "episodeNotFound":
case "encounterNotFound":
// No pre-links are passed on this path; unreachable, but a result
// the switch does not name must not fall through silently.
return sourceErrorResponse("linkTargetNotFound");
}
}
src/app/api/documents/sources/[system]/route.ts
+19 -6
View File
@@ -6,8 +6,9 @@
* (`DOCUMENT_SOURCE_ORIGINS`), tests the connection live before storing
* anything (a wrong token or an unreachable host is reported here, not on the
* first search), then stores the token encrypted. A PUT without a token keeps
* the stored one, so the address or the Papra organization can change without
* typing the key again; the first save must carry it. The response is the
* the stored one, so the base path or the Papra organization can change
* without typing the key again, but only on the same origin: a new origin, and
* the first save, must carry it (422 `documents.sources.tokenRequired`). The response is the
* connection without its token, like every read of it.
*
* DELETE removes the row, and with it the token.
@@ -100,8 +101,16 @@ export const PUT = apiHandler(
}
const existing = await loadConnection(userId, system);
// The stored token is only ever sent back to the origin it was saved
// for. A new address on another origin needs the token typed again, or
// saving an address would hand the old archive's token to whatever the
// new one is.
const reusable =
existing !== null &&
evaluateSourceBaseUrl(existing.baseUrl, new Set([verdict.origin])).ok;
const token =
parsed.data.token ?? (existing ? decrypt(existing.tokenEncrypted) : null);
parsed.data.token ??
(existing && reusable ? decrypt(existing.tokenEncrypted) : null);
if (!token) {
return apiError("Enter the API token.", 422, {
errorCode: "documents.sources.tokenRequired",
@@ -165,9 +174,13 @@ export const PUT = apiHandler(
export const DELETE = apiHandler(
async (request: NextRequest, { params }: RouteParams) => {
const auth = await requireAuth();
// No list needed to forget a connection: an operator who removed the
// origin must not leave the person unable to delete the stored token.
const refused = await admitDocumentSourceCaller(auth, { needsList: false });
// Neither the list nor the module is needed to forget a connection: an
// operator who removed the origin, or a person who switched Documents
// off, must still be able to delete the stored token.
const refused = await admitDocumentSourceCaller(auth, {
needsList: false,
needsModule: false,
});
if (refused) return refused;
const system = await systemParam(params);
if (system instanceof Response) return system;
src/app/api/documents/sources/[system]/search/route.ts
+41 -13
View File
@@ -9,6 +9,10 @@
* `new`, `imported` (with the vault document's id) or `deleted` (the person
* deleted it here; importing it again is refused). Nothing is stored.
*
* When the source ignores a filter and the local re-check empties a page, the
* route reads on (up to five source pages) and answers with the last page it
* read, so the client's "load more" continues after it.
*
* Sixty searches a minute, shared with the tag read. Cookie-only and
* owner-only; see `admitDocumentSourceCaller`.
*/
@@ -30,9 +34,10 @@ import {
sourceErrorResponse,
systemParam,
} from "@/lib/documents/sources/route-support";
import type {
DocumentSourceResultDto,
DocumentSourceSearchDto,
import {
DOCUMENT_PICKER_MAX_PAGE,
type DocumentSourceResultDto,
type DocumentSourceSearchDto,
} from "@/lib/documents/sources/types";
import { annotate } from "@/lib/logging/context";
import { rateLimitHeaders } from "@/lib/rate-limit";
@@ -40,6 +45,9 @@ import { documentSourceSearchSchema } from "@/lib/validations/document-sources";
export const dynamic = "force-dynamic";
/** Source pages one search request may read to find a non-empty one. */
const SEARCH_PAGES_PER_REQUEST = 5;
type RouteParams = { params: Promise<{ system: string }> };
export const GET = apiHandler(
@@ -77,16 +85,34 @@ export const GET = apiHandler(
);
}
let found: { items: SourceListItem[]; hasMore: boolean };
// A page the local tag/date re-check emptied is not an answer: the next
// source page may hold matches. Read on, a bounded number of pages, and
// report the last one read so "load more" continues after it.
let found: { items: SourceListItem[]; hasMore: boolean } = {
items: [],
hasMore: false,
};
let lastPage = page;
let instance: string;
try {
const client = await openConnection(userId, system);
found = await client.search({
q,
tagId: tag ?? null,
from: from ?? null,
to: to ?? null,
page,
});
instance = client.instance;
for (
let next = page;
next < page + SEARCH_PAGES_PER_REQUEST &&
next <= DOCUMENT_PICKER_MAX_PAGE;
next += 1
) {
found = await client.search({
q,
tagId: tag ?? null,
from: from ?? null,
to: to ?? null,
page: next,
});
lastPage = next;
if (found.items.length > 0 || !found.hasMore) break;
}
} catch (err) {
return responseForSourceFailure(err);
}
@@ -95,6 +121,7 @@ export const GET = apiHandler(
userId,
system,
found.items.map((item) => item.sourceId),
instance,
);
const results: DocumentSourceResultDto[] = found.items.map((item) => {
const held = states.get(item.sourceId);
@@ -116,6 +143,7 @@ export const GET = apiHandler(
meta: {
system,
page,
pagesRead: lastPage - page + 1,
count: results.length,
filtered: Boolean(q || tag || from || to),
},
@@ -123,8 +151,8 @@ export const GET = apiHandler(
const body: DocumentSourceSearchDto = {
results,
page,
hasMore: found.hasMore,
page: lastPage,
hasMore: found.hasMore && lastPage < DOCUMENT_PICKER_MAX_PAGE,
};
return apiSuccess(body);
},
src/app/api/documents/sources/route.ts
+13 -6
View File
@@ -8,9 +8,10 @@
* No request leaves the server here.
*
* Cookie-only and owner-only like every route under this path
* (`admitDocumentSourceCaller`). With the list unset it answers 200
* `available: false` rather than 404, because this is the one read the client
* makes to decide whether to show the picker at all.
* (`admitDocumentSourceCaller`), but it needs neither the documents module nor
* the list: with the list unset it answers 200 `available: false` (plus any
* rows still stored, so they can be deleted) rather than 404, because this is
* the one read the client makes to decide what to show.
*/
import { apiHandler, requireAuth } from "@/lib/api-handler";
import { apiSuccess } from "@/lib/api-response";
@@ -30,21 +31,27 @@ export const dynamic = "force-dynamic";
export const GET = apiHandler(async () => {
const auth = await requireAuth();
const refused = await admitDocumentSourceCaller(auth, { needsList: false });
const refused = await admitDocumentSourceCaller(auth, {
needsList: false,
needsModule: false,
});
if (refused) return refused;
annotate({ action: { name: "documents.sources.status" } });
// Rows are listed even with the picker off: a connection whose origin the
// operator removed (or whose list is gone) must stay visible so it can be
// deleted. `originAllowed` is false for each of them then.
const rows = await listConnections(auth.user.id);
if (!documentSourcesAvailable()) {
const off: DocumentSourcesStatusDto = {
available: false,
allowedOrigins: [],
connections: [],
connections: rows.map(toConnectionDto),
};
return apiSuccess(off);
}
const rows = await listConnections(auth.user.id);
const status: DocumentSourcesStatusDto = {
available: true,
allowedOrigins: listedDocumentSourceOrigins(),
src/lib/documents/ingest.ts
+20 -2
View File
@@ -93,6 +93,11 @@ export interface IngestInput {
encounterIds: string[];
sourceSystem: DocumentSourceSystemValue | null;
sourceId: string | null;
/**
* The source instance (origin) of the key, or null for a key sent without
* one; see `src/lib/documents/source-key.ts` for how null matches.
*/
sourceInstance: string | null;
/** Hold back automatic AI reading (`aiRead=defer`). */
aiDeferred: boolean;
/**
@@ -151,6 +156,7 @@ async function answerDuplicate(
existing.id,
input.sourceSystem,
input.sourceId,
input.sourceInstance,
);
if (remembered === "limit") return { kind: "aliasLimit" };
}
@@ -161,6 +167,7 @@ export async function ingestDocument(
input: IngestInput,
): Promise<IngestResult> {
const { userId, sourceSystem, sourceId, limits, bytes } = input;
const sourceInstance = sourceId ? input.sourceInstance : null;
// A source key answers before the bytes are looked at: an import re-sending
// what it sent before is a duplicate, and one re-sending a document the
@@ -168,7 +175,12 @@ export async function ingestDocument(
// count (the unique index has no `deleted_at` predicate), and past the purge
// the ledger remembers.
if (sourceSystem && sourceId && !input.sourceKeyChecked) {
const match = await findSourceKey(userId, sourceSystem, sourceId);
const match = await findSourceKey(
userId,
sourceSystem,
sourceId,
sourceInstance,
);
if (match) return { kind: "sourceKey", match };
}
@@ -258,6 +270,7 @@ export async function ingestDocument(
: new Date(),
sourceSystem,
sourceId,
sourceInstance,
aiReadDeferred: input.aiDeferred,
},
omit: { contentEncrypted: true },
@@ -296,7 +309,12 @@ export async function ingestDocument(
// source key, or the same bytes. Surface the winner exactly as the fast
// paths above would have.
if (sourceSystem && sourceId) {
const match = await findSourceKey(userId, sourceSystem, sourceId);
const match = await findSourceKey(
userId,
sourceSystem,
sourceId,
sourceInstance,
);
if (match) return { kind: "sourceKey", match };
}
const winner = await prisma.inboundDocument.findFirst({
src/lib/documents/source-key.ts
+75 -18
View File
@@ -11,8 +11,21 @@
* 3. `DocumentImportKey` — a key whose document the purge has removed.
*
* The first two resolve to a document (live → duplicate, tombstoned →
* deleted); the third only to "deleted". The upload route and the lookup
* route both ask here, so they cannot disagree about what a key means.
* deleted); the third only to "deleted". The upload route, the lookup route
* and the document picker all ask here, so they cannot disagree about what a
* key means.
*
* v1.39.3 — a key also names its instance (`sourceInstance`, the origin of
* the Paperless-ngx or Papra it came from): id 123 on one Paperless and id
* 123 on another are different documents. A null on either side is a
* wildcard, so nothing stored before the column existed is forgotten:
*
* - a key stored without an instance (a v1.39.2 import, an older script, a
* workflow that does not send one) matches a lookup for any instance of
* its system;
* - a lookup without an instance matches a key stored for any instance.
*
* Where both an exact and a wildcard match exist, the exact one wins.
*/
import { prisma } from "@/lib/db";
import { checkRateLimit, type RateLimitResult } from "@/lib/rate-limit";
@@ -23,22 +36,44 @@ export type SourceKeyMatch =
| { state: "live"; document: SerialisableDocument }
| { state: "deleted"; id: string | null };
/**
* The instance half of a key match: the same instance or a stored key without
* one; any instance when the lookup has none.
*/
export function instanceMatch(sourceInstance: string | null): {
OR?: Array<{ sourceInstance: string | null }>;
} {
return sourceInstance === null
? {}
: { OR: [{ sourceInstance }, { sourceInstance: null }] };
}
/** Exact instance first, then a key stored without one (nulls sort last). */
const EXACT_FIRST = { sourceInstance: { sort: "asc", nulls: "last" } } as const;
export async function findSourceKey(
userId: string,
sourceSystem: DocumentSourceSystemValue,
sourceId: string,
sourceInstance: string | null = null,
): Promise<SourceKeyMatch | null> {
const key = {
userId,
sourceSystem,
sourceId,
...instanceMatch(sourceInstance),
};
const own = await prisma.inboundDocument.findFirst({
where: { userId, sourceSystem, sourceId },
where: key,
omit: { contentEncrypted: true },
orderBy: EXACT_FIRST,
});
const viaAlias = own
? null
: await prisma.documentSourceAlias.findUnique({
where: {
userId_sourceSystem_sourceId: { userId, sourceSystem, sourceId },
},
: await prisma.documentSourceAlias.findFirst({
where: key,
select: { document: { omit: { contentEncrypted: true } } },
orderBy: EXACT_FIRST,
});
const document = own ?? viaAlias?.document ?? null;
if (document) {
@@ -46,10 +81,8 @@ export async function findSourceKey(
? { state: "deleted", id: document.id }
: { state: "live", document };
}
const purged = await prisma.documentImportKey.findUnique({
where: {
userId_sourceSystem_sourceId: { userId, sourceSystem, sourceId },
},
const purged = await prisma.documentImportKey.findFirst({
where: key,
select: { id: true },
});
return purged ? { state: "deleted", id: null } : null;
@@ -74,13 +107,14 @@ export async function rememberSourceAlias(
documentId: string,
sourceSystem: DocumentSourceSystemValue,
sourceId: string,
sourceInstance: string | null = null,
): Promise<"remembered" | "known" | "limit"> {
const held = await prisma.documentSourceAlias.count({
where: { userId, documentId },
});
if (held >= MAX_SOURCE_ALIASES_PER_DOCUMENT) return "limit";
const { count } = await prisma.documentSourceAlias.createMany({
data: [{ userId, documentId, sourceSystem, sourceId }],
data: [{ userId, documentId, sourceSystem, sourceInstance, sourceId }],
skipDuplicates: true,
});
return count > 0 ? "remembered" : "known";
@@ -123,33 +157,56 @@ export async function findSourceKeyStates(
userId: string,
sourceSystem: DocumentSourceSystemValue,
sourceIds: string[],
sourceInstance: string | null = null,
): Promise<Map<string, SourceKeyState>> {
const states = new Map<string, SourceKeyState>();
const ids = [...new Set(sourceIds)];
if (ids.length === 0) return states;
const where = {
userId,
sourceSystem,
sourceId: { in: ids },
...instanceMatch(sourceInstance),
};
const [own, aliases, purged] = await Promise.all([
prisma.inboundDocument.findMany({
where: { userId, sourceSystem, sourceId: { in: ids } },
select: { id: true, sourceId: true, deletedAt: true },
where,
select: {
id: true,
sourceId: true,
sourceInstance: true,
deletedAt: true,
},
}),
prisma.documentSourceAlias.findMany({
where: { userId, sourceSystem, sourceId: { in: ids } },
where,
select: {
sourceId: true,
sourceInstance: true,
document: { select: { id: true, deletedAt: true } },
},
}),
prisma.documentImportKey.findMany({
where: { userId, sourceSystem, sourceId: { in: ids } },
where,
select: { sourceId: true },
}),
]);
// A wildcard (instance-less) row first, so an exact-instance row after it
// wins, as it does in `findSourceKey`.
const wildcardFirst = <T extends { sourceInstance: string | null }>(
rows: T[],
) =>
[...rows].sort(
(a, b) =>
Number(a.sourceInstance === sourceInstance) -
Number(b.sourceInstance === sourceInstance),
);
// Lowest precedence first, so the document's own key wins, as it does in
// `findSourceKey`.
for (const row of purged) states.set(row.sourceId, { state: "deleted" });
for (const row of aliases) {
for (const row of wildcardFirst(aliases)) {
states.set(
row.sourceId,
row.document.deletedAt
@@ -157,7 +214,7 @@ export async function findSourceKeyStates(
: { state: "imported", documentId: row.document.id },
);
}
for (const row of own) {
for (const row of wildcardFirst(own)) {
if (!row.sourceId) continue;
states.set(
row.sourceId,
src/lib/documents/sources/__tests__/clients.test.ts
+74 -2
View File
@@ -171,8 +171,10 @@ describe("Paperless-ngx", () => {
const list = calledUrls().find((u) => u.pathname.endsWith("/documents/"));
expect(list?.searchParams.get("title__icontains")).toBe("blood");
expect(list?.searchParams.get("tags__id__all")).toBe("7");
expect(list?.searchParams.get("created__date__gte")).toBe("2025-01-01");
expect(list?.searchParams.get("created__date__lte")).toBe("2025-12-31");
expect(list?.searchParams.get("created__gte")).toBe("2025-01-01");
expect(list?.searchParams.get("created__lte")).toBe("2025-12-31");
// API version 9 made `created` a date; the `__date` lookups are gone.
expect(list?.searchParams.has("created__date__gte")).toBe(false);
expect(list?.searchParams.get("page")).toBe("2");
expect(list?.searchParams.get("page_size")).toBe("25");
@@ -311,6 +313,76 @@ describe("Paperless-ngx", () => {
});
});
describe("ids never shape a request path", () => {
it("refuses a Paperless id that is not a number, and dot segments, before dialling", async () => {
const client = sourceClient(PAPERLESS);
for (const id of ["..", ".", "41/../x", "abc", "41?x=1"]) {
const err = await failure(client.document(id));
expect((err as DocumentSourceError).code, id).toBe("notFound");
expect(
((await failure(client.download(id, 10))) as DocumentSourceError).code,
).toBe("notFound");
}
expect(safeFetch).not.toHaveBeenCalled();
});
it("refuses a Papra id or organization id outside [A-Za-z0-9_-]", async () => {
for (const id of ["..", "doc/1", "doc%2F1", "doc 1"]) {
await failure(sourceClient(PAPRA).document(id));
}
expect(() =>
sourceClient({ ...PAPRA, organizationId: "../admin" }),
).toThrow(DocumentSourceError);
expect(safeFetch).not.toHaveBeenCalled();
});
it("leaves a result with an unusable id out of the list", async () => {
serve({
"/paperless/api/tags/": () => json({ results: [], next: null }),
"/paperless/api/documents/": () =>
json({
next: null,
results: [
{ id: "..", title: "Odd", created: "2025-01-01", tags: [] },
{ id: 5, title: "Fine", created: "2025-01-01", tags: [] },
],
}),
});
const result = await sourceClient(PAPERLESS).search({
q: "",
tagId: null,
from: null,
to: null,
page: 1,
});
expect(result.items.map((i) => i.sourceId)).toEqual(["5"]);
});
});
describe("Paperless-ngx tags", () => {
it("pages through tags beyond the first 250", async () => {
serve({
"/paperless/api/tags/": (url) => {
const page = Number(url.searchParams.get("page"));
const results = Array.from(
{ length: page === 1 ? 250 : 3 },
(_, i) => ({
id: page * 1000 + i,
name: `t${page}-${i}`,
}),
);
return json({ results, next: page === 1 ? "more" : null });
},
});
const tags = await sourceClient(PAPERLESS).tags();
expect(tags).toHaveLength(253);
expect(calledUrls().map((u) => u.searchParams.get("page"))).toEqual([
"1",
"2",
]);
});
});
describe("Papra", () => {
const tags = {
tags: [
src/lib/documents/sources/clients.ts
+55 -21
View File
@@ -29,6 +29,8 @@ import {
import {
DOCUMENT_PICKER_MAX_TAGS,
DOCUMENT_PICKER_PAGE_SIZE,
isSourceDocumentId,
PAPRA_ID,
type DocumentPickerSystem,
type DocumentSourceTagDto,
} from "./types";
@@ -64,6 +66,11 @@ export interface SourceDocumentMeta {
export interface SourceClient {
readonly system: DocumentPickerSystem;
/**
* The instance this client talks to, as the normalised origin every source
* key it produces carries (`sourceInstance`).
*/
readonly instance: string;
test(): Promise<void>;
tags(): Promise<DocumentSourceTagDto[]>;
search(
@@ -84,6 +91,16 @@ export interface SourceConnection {
token: string;
}
/** Paperless tags are read this many per page. */
const TAG_PAGE_SIZE = 250;
/** Refuse an id that is not one the system issues, before it reaches a URL. */
function requireId(system: DocumentPickerSystem, id: string): void {
if (!isSourceDocumentId(system, id)) {
throw new DocumentSourceError("notFound");
}
}
// ── Defensive readers ──────────────────────────────────────────────────────
type Json = Record<string, unknown>;
@@ -181,26 +198,36 @@ function paperless(connection: SourceConnection): SourceClient {
}
async function listTags(): Promise<DocumentSourceTagDto[]> {
const query = new URLSearchParams({
page_size: String(DOCUMENT_PICKER_MAX_TAGS),
ordering: "name",
});
const { body, response } = await sourceJson(target, `/api/tags/?${query}`);
checkVersion(response);
const page = asRecord(body);
if (!page) throw shapeError();
// Paged by number rather than by following `next`: the link is the
// server's to write, the page number is ours.
const tags: DocumentSourceTagDto[] = [];
for (const raw of asArray(page.results)) {
const tag = asRecord(raw);
const id = asText(tag?.id);
const name = asText(tag?.name);
if (id && name) tags.push({ id, name });
for (let pageNo = 1; tags.length < DOCUMENT_PICKER_MAX_TAGS; pageNo += 1) {
const query = new URLSearchParams({
page: String(pageNo),
page_size: String(TAG_PAGE_SIZE),
ordering: "name",
});
const { body, response } = await sourceJson(
target,
`/api/tags/?${query}`,
);
checkVersion(response);
const page = asRecord(body);
if (!page) throw shapeError();
for (const raw of asArray(page.results)) {
const tag = asRecord(raw);
const id = asText(tag?.id);
const name = asText(tag?.name);
if (id && name) tags.push({ id, name });
}
if (typeof page.next !== "string" || !page.next) break;
}
return tags;
return tags.slice(0, DOCUMENT_PICKER_MAX_TAGS);
}
return {
system: "PAPERLESS",
instance: connection.origin,
async test() {
const { body, response } = await sourceJson(
@@ -223,8 +250,10 @@ function paperless(connection: SourceConnection): SourceClient {
});
if (params.q) query.set("title__icontains", params.q);
if (params.tagId) query.set("tags__id__all", params.tagId);
if (params.from) query.set("created__date__gte", params.from);
if (params.to) query.set("created__date__lte", params.to);
// API version 9 made `created` a plain date, so the range filters on
// it directly (`created__gte` / `created__lte`), both ends inclusive.
if (params.from) query.set("created__gte", params.from);
if (params.to) query.set("created__lte", params.to);
let page: Json | null;
let names: Map<string, string>;
@@ -250,7 +279,7 @@ function paperless(connection: SourceConnection): SourceClient {
for (const raw of asArray(page.results)) {
const doc = asRecord(raw);
const id = asText(doc?.id);
if (!doc || !id) continue;
if (!doc || !id || !isSourceDocumentId("PAPERLESS", id)) continue;
const tagIds = asArray(doc.tags)
.map(asText)
.filter((t): t is string => t !== null);
@@ -275,6 +304,7 @@ function paperless(connection: SourceConnection): SourceClient {
},
async document(sourceId) {
requireId("PAPERLESS", sourceId);
const { body, response } = await sourceJson(
target,
`/api/documents/${encodeURIComponent(sourceId)}/?fields=id,title,created,added,original_file_name`,
@@ -290,7 +320,8 @@ function paperless(connection: SourceConnection): SourceClient {
};
},
download(sourceId, maxFileBytes) {
async download(sourceId, maxFileBytes) {
requireId("PAPERLESS", sourceId);
// The original, not the archived PDF/A: the same bytes a Paperless
// workflow or the import script sends, so they dedup against each other.
return sourceDownload(
@@ -310,7 +341,7 @@ function papraTagTerm(name: string): string {
}
function papra(connection: SourceConnection): SourceClient {
if (!connection.organizationId) {
if (!connection.organizationId || !PAPRA_ID.test(connection.organizationId)) {
throw new DocumentSourceError("notFound");
}
const org = `/api/organizations/${encodeURIComponent(connection.organizationId)}`;
@@ -345,6 +376,7 @@ function papra(connection: SourceConnection): SourceClient {
return {
system: "PAPRA",
instance: connection.origin,
async test() {
const { body } = await sourceJson(
@@ -381,7 +413,7 @@ function papra(connection: SourceConnection): SourceClient {
for (const raw of documents) {
const doc = asRecord(raw);
const id = asText(doc?.id);
if (!doc || !id) continue;
if (!doc || !id || !isSourceDocumentId("PAPRA", id)) continue;
const tags = asArray(doc.tags)
.map(asRecord)
.filter((t): t is Json => t !== null);
@@ -417,6 +449,7 @@ function papra(connection: SourceConnection): SourceClient {
},
async document(sourceId) {
requireId("PAPRA", sourceId);
const { body } = await sourceJson(
target,
`${org}/documents/${encodeURIComponent(sourceId)}`,
@@ -435,7 +468,8 @@ function papra(connection: SourceConnection): SourceClient {
};
},
download(sourceId, maxFileBytes) {
async download(sourceId, maxFileBytes) {
requireId("PAPRA", sourceId);
return sourceDownload(
download,
`${org}/documents/${encodeURIComponent(sourceId)}/file`,
src/lib/documents/sources/route-support.ts
+11 -5
View File
@@ -39,12 +39,13 @@ import {
* Each route calls `requireAuth()` itself, in its own module, so the sharing
* guard's scan sees every one of them as a bare, owner-only route rather than
* a helper it cannot follow. This adds the rest: a cookie transport, the
* documents module, and (unless `needsList` is false, which only the status
* read passes) the operator's list.
* documents module and the operator's list. The status read and the delete
* pass `needsList: false, needsModule: false`: they touch only the person's
* own row and never the network.
*/
export async function admitDocumentSourceCaller(
auth: AuthContext,
options: { needsList?: boolean } = {},
options: { needsList?: boolean; needsModule?: boolean } = {},
): Promise<Response | null> {
if (auth.authMethod !== "cookie") {
return apiError(
@@ -53,8 +54,13 @@ export async function admitDocumentSourceCaller(
{ errorCode: "documents.sources.browserOnly" },
);
}
const gate = await requireModuleEnabled(auth.user.id, "inboundDocuments");
if (!gate.enabled) return gate.response;
// Reading and removing one's own connection needs neither the module nor
// the list: a person who switched Documents off, or whose operator removed
// the list, must still be able to see and delete the stored token.
if (options.needsModule !== false) {
const gate = await requireModuleEnabled(auth.user.id, "inboundDocuments");
if (!gate.enabled) return gate.response;
}
if (options.needsList !== false && !documentSourcesAvailable()) {
return sourceErrorResponse("unavailable");
}
src/lib/documents/sources/types.ts
+20 -2
View File
@@ -38,8 +38,26 @@ export const DOCUMENT_PICKER_MAX_PAGE = 500;
export const DOCUMENT_PICKER_QUERY_MAX = 200;
/** Documents the sheet imports in one run. */
export const DOCUMENT_PICKER_MAX_SELECTION = 25;
/** Tags offered in the filter. */
export const DOCUMENT_PICKER_MAX_TAGS = 250;
/** Tags offered in the filter (read in pages of 250). */
export const DOCUMENT_PICKER_MAX_TAGS = 1000;
/** Paperless-ngx document ids are integers. */
export const PAPERLESS_ID = /^\d{1,18}$/;
/** Papra document and organization ids: letters, digits, `_` and `-`. */
export const PAPRA_ID = /^[A-Za-z0-9_-]{1,128}$/;
/**
* Is `id` a document id `system` could have issued? Checked before an id is
* put into a request path, so nothing a request carries can add a path
* segment, a query or a `..` to the URL built from it.
*/
export function isSourceDocumentId(
system: DocumentPickerSystem,
id: string,
): boolean {
if (id === "." || id === "..") return false;
return (system === "PAPERLESS" ? PAPERLESS_ID : PAPRA_ID).test(id);
}
/** A connection as every route may show it: never the token. */
export interface DocumentSourceConnectionDto {
src/lib/export/records-backup.ts
+5
View File
@@ -177,6 +177,8 @@ export interface DocumentBackupEntry {
*/
sourceSystem?: string | null;
sourceId?: string | null;
/** v1.39.3 — the source instance (origin); absent in older files. */
sourceInstance?: string | null;
/** Canonical DR only: the import's hold on automatic AI reading. */
aiReadDeferred?: boolean;
}
@@ -309,6 +311,7 @@ export async function buildRecordsBackupSection(
lastIndexOutcome: true,
sourceSystem: true,
sourceId: true,
sourceInstance: true,
aiReadDeferred: true,
createdAt: true,
updatedAt: true,
@@ -330,6 +333,7 @@ export async function buildRecordsBackupSection(
summaryEncrypted: true,
sourceSystem: true,
sourceId: true,
sourceInstance: true,
createdAt: true,
},
});
@@ -545,6 +549,7 @@ export async function buildRecordsBackupSection(
: null,
sourceSystem: d.sourceSystem,
sourceId: d.sourceId,
sourceInstance: d.sourceInstance,
createdAt: d.createdAt.toISOString(),
};
src/lib/export/restore-backup.ts
+9
View File
@@ -30,6 +30,7 @@
import { Buffer } from "node:buffer";
import { prisma, toJson } from "@/lib/db";
import { normaliseSourceInstance } from "@/lib/validations/inbound-documents";
import { auditLog } from "@/lib/auth/audit";
import {
isStoredBackupReadError,
@@ -2057,6 +2058,14 @@ export async function restoreBackup(
sourceId: document.sourceSystem
? (document.sourceId ?? null)
: null,
// A file written before v1.39.3 has no instance: the key then
// matches any instance of its system, as it did when stored.
sourceInstance:
document.sourceSystem && document.sourceId
? document.sourceInstance
? normaliseSourceInstance(document.sourceInstance)
: null
: null,
createdAt: new Date(document.createdAt!),
updatedAt: new Date(document.updatedAt!),
})),
src/lib/jobs/document-purge.ts
+6 -1
View File
@@ -58,9 +58,12 @@ export async function purgeExpiredDocumentTombstones(
userId: true,
sourceSystem: true,
sourceId: true,
sourceInstance: true,
// Further keys the import answered with this document (same bytes).
// They cascade away with the row, so they are copied here first.
sourceAliases: { select: { sourceSystem: true, sourceId: true } },
sourceAliases: {
select: { sourceSystem: true, sourceId: true, sourceInstance: true },
},
},
});
const keys = keyed.flatMap((row) => [
@@ -70,6 +73,7 @@ export async function purgeExpiredDocumentTombstones(
userId: row.userId,
sourceSystem: row.sourceSystem,
sourceId: row.sourceId,
sourceInstance: row.sourceInstance,
},
]
: []),
@@ -77,6 +81,7 @@ export async function purgeExpiredDocumentTombstones(
userId: row.userId,
sourceSystem: alias.sourceSystem,
sourceId: alias.sourceId,
sourceInstance: alias.sourceInstance,
})),
]);
if (keys.length > 0) {
src/lib/validations/backup.ts
+9
View File
@@ -83,6 +83,7 @@ import {
} from "@/lib/validations/emergency-profile";
import {
DOCUMENT_SOURCE_ID_MAX,
DOCUMENT_SOURCE_INSTANCE_MAX,
DOCUMENT_SOURCE_SYSTEMS,
} from "@/lib/validations/inbound-documents";
import { REMINDER_EVENT_SOURCES } from "@/lib/measurement-reminders/satisfy";
@@ -1636,6 +1637,14 @@ const documentBackupSchema = z
// restore cannot write a system the detail sheet has no name for.
sourceSystem: z.enum(DOCUMENT_SOURCE_SYSTEMS).nullable().optional(),
sourceId: z.string().max(DOCUMENT_SOURCE_ID_MAX).nullable().optional(),
// v1.39.3 — the source instance (an origin). Length-bounded here and
// normalised on restore; a value that is not an origin restores as null
// (matches any instance) rather than failing the restore.
sourceInstance: z
.string()
.max(DOCUMENT_SOURCE_INSTANCE_MAX)
.nullable()
.optional(),
aiReadDeferred: z.boolean().optional(),
summary: z.string().nullable().optional(),
createdAt: isoDateTime.optional(),
src/lib/validations/document-sources.ts
+2 -3
View File
@@ -10,6 +10,7 @@ import {
DOCUMENT_PICKER_LINK_KINDS,
DOCUMENT_PICKER_MAX_PAGE,
DOCUMENT_PICKER_QUERY_MAX,
PAPRA_ID,
} from "@/lib/documents/sources/types";
import {
documentSourceIdSchema,
@@ -37,9 +38,7 @@ export const documentSourceSaveSchema = z.object({
organizationId: z
.string()
.trim()
.min(1)
.max(128)
.regex(printable, "Expected printable characters without spaces")
.regex(PAPRA_ID, "Expected letters, digits, _ or -")
.optional(),
token: z
.string()
src/lib/validations/inbound-documents.ts
+46 -1
View File
@@ -706,7 +706,41 @@ export const documentSourceIdSchema = z
.trim()
.min(1)
.max(DOCUMENT_SOURCE_ID_MAX)
.regex(/^[\x21-\x7e]+$/u, "Expected printable characters without spaces");
.regex(/^[\x21-\x7e]+$/u, "Expected printable characters without spaces")
// A path segment of its own name: an importer that builds a URL from the id
// must never be handed one that walks up the source's path.
.refine((v) => v !== "." && v !== "..", "Not a document id");
/** Longest source instance (an origin) a key may carry. */
export const DOCUMENT_SOURCE_INSTANCE_MAX = 512;
/**
* v1.39.3 — the instance of the source system a key belongs to, as the
* normalised origin (`scheme://host[:port]`) of the Paperless-ngx or Papra it
* came from, or null when the value is not one. A path, query or credentials
* in the value are dropped: two addresses on one origin are one instance.
*/
export function normaliseSourceInstance(value: string): string | null {
try {
const url = new URL(value.trim());
if (url.protocol !== "http:" && url.protocol !== "https:") return null;
if (url.username || url.password) return null;
return url.origin;
} catch {
return null;
}
}
/** The instance as a request field: validated, then normalised to its origin. */
export const documentSourceInstanceSchema = z
.string()
.trim()
.min(1)
.max(DOCUMENT_SOURCE_INSTANCE_MAX)
.refine((v) => normaliseSourceInstance(v) !== null, {
message: "Expected the http(s) address of the source instance",
})
.transform((v) => normaliseSourceInstance(v) as string);
/**
* A source key on its own — the query-string form the upload accepts ahead of
@@ -715,6 +749,8 @@ export const documentSourceIdSchema = z
export const documentSourceKeySchema = z.object({
sourceSystem: z.enum(DOCUMENT_SOURCE_SYSTEMS),
sourceId: documentSourceIdSchema,
/** Optional: without it the key matches any instance of the system. */
sourceInstance: documentSourceInstanceSchema.optional(),
});
export const documentCreateSchema = z
@@ -731,6 +767,11 @@ export const documentCreateSchema = z
* not prose, and it is echoed on the detail sheet.
*/
sourceId: documentSourceIdSchema.optional(),
/**
* v1.39.3 — which instance of `sourceSystem` (its origin). Optional; a
* key without one matches any instance.
*/
sourceInstance: documentSourceInstanceSchema.optional(),
/**
* `defer` holds back automatic AI reading for this upload: the thumbnail
* and a local text index still run, the summary and the lab staging do
@@ -741,6 +782,10 @@ export const documentCreateSchema = z
.refine((v) => v.sourceId === undefined || v.sourceSystem !== undefined, {
path: ["sourceSystem"],
message: "sourceId needs a sourceSystem",
})
.refine((v) => v.sourceInstance === undefined || v.sourceId !== undefined, {
path: ["sourceInstance"],
message: "sourceInstance needs a sourceId",
});
export type DocumentCreateInput = z.infer<typeof documentCreateSchema>;