mirror of
https://github.com/MBombeck/HealthLog.git
synced 2026-11-17 23:52:24 +00:00
fix(documents): key imports by source instance, keep a token on its origin, and harden the import path
- Source keys carry the normalised origin of the Paperless-ngx or Papra they came from (source_instance on the document, the alias and the purge ledger; unique NULLS NOT DISTINCT). A key stored without one (v1.39.2 imports) matches any instance of its system, and a lookup without one matches any stored instance. The upload route, the lookup route and the import script accept or send it; backups carry it. - Saving a connection reuses the stored token only for the same origin; a new origin needs the token again (422 tokenRequired). - Document ids are checked against what each system issues (Paperless digits, Papra [A-Za-z0-9_-], never . or ..) before any URL is built; the Papra organization id likewise. - Paperless API 9 filters dates on created__gte / created__lte, in the picker and the script. A search page the local re-check empties reads on, up to five source pages. Tags are read past the first 250. - The import's key pre-check uses the lookup allowance, a failed fetch no longer charges the upload allowance, and one import per key runs at a time under an advisory lock, so a double click downloads once. - Reading and deleting one's own connection works with the documents module off and the operator list unset.
This commit is contained in:
1 parent
6029c7c30d
commit
1eac05b454
24 files changed
+886
-209
No files matched your search
@@ -93,6 +93,11 @@ export interface IngestInput {
|
||||
encounterIds: string[];
|
||||
sourceSystem: DocumentSourceSystemValue | null;
|
||||
sourceId: string | null;
|
||||
/**
|
||||
* The source instance (origin) of the key, or null for a key sent without
|
||||
* one; see `src/lib/documents/source-key.ts` for how null matches.
|
||||
*/
|
||||
sourceInstance: string | null;
|
||||
/** Hold back automatic AI reading (`aiRead=defer`). */
|
||||
aiDeferred: boolean;
|
||||
/**
|
||||
@@ -151,6 +156,7 @@ async function answerDuplicate(
|
||||
existing.id,
|
||||
input.sourceSystem,
|
||||
input.sourceId,
|
||||
input.sourceInstance,
|
||||
);
|
||||
if (remembered === "limit") return { kind: "aliasLimit" };
|
||||
}
|
||||
@@ -161,6 +167,7 @@ export async function ingestDocument(
|
||||
input: IngestInput,
|
||||
): Promise<IngestResult> {
|
||||
const { userId, sourceSystem, sourceId, limits, bytes } = input;
|
||||
const sourceInstance = sourceId ? input.sourceInstance : null;
|
||||
|
||||
// A source key answers before the bytes are looked at: an import re-sending
|
||||
// what it sent before is a duplicate, and one re-sending a document the
|
||||
@@ -168,7 +175,12 @@ export async function ingestDocument(
|
||||
// count (the unique index has no `deleted_at` predicate), and past the purge
|
||||
// the ledger remembers.
|
||||
if (sourceSystem && sourceId && !input.sourceKeyChecked) {
|
||||
const match = await findSourceKey(userId, sourceSystem, sourceId);
|
||||
const match = await findSourceKey(
|
||||
userId,
|
||||
sourceSystem,
|
||||
sourceId,
|
||||
sourceInstance,
|
||||
);
|
||||
if (match) return { kind: "sourceKey", match };
|
||||
}
|
||||
|
||||
@@ -258,6 +270,7 @@ export async function ingestDocument(
|
||||
: new Date(),
|
||||
sourceSystem,
|
||||
sourceId,
|
||||
sourceInstance,
|
||||
aiReadDeferred: input.aiDeferred,
|
||||
},
|
||||
omit: { contentEncrypted: true },
|
||||
@@ -296,7 +309,12 @@ export async function ingestDocument(
|
||||
// source key, or the same bytes. Surface the winner exactly as the fast
|
||||
// paths above would have.
|
||||
if (sourceSystem && sourceId) {
|
||||
const match = await findSourceKey(userId, sourceSystem, sourceId);
|
||||
const match = await findSourceKey(
|
||||
userId,
|
||||
sourceSystem,
|
||||
sourceId,
|
||||
sourceInstance,
|
||||
);
|
||||
if (match) return { kind: "sourceKey", match };
|
||||
}
|
||||
const winner = await prisma.inboundDocument.findFirst({
|
||||
|
||||
@@ -11,8 +11,21 @@
|
||||
* 3. `DocumentImportKey` — a key whose document the purge has removed.
|
||||
*
|
||||
* The first two resolve to a document (live → duplicate, tombstoned →
|
||||
* deleted); the third only to "deleted". The upload route and the lookup
|
||||
* route both ask here, so they cannot disagree about what a key means.
|
||||
* deleted); the third only to "deleted". The upload route, the lookup route
|
||||
* and the document picker all ask here, so they cannot disagree about what a
|
||||
* key means.
|
||||
*
|
||||
* v1.39.3 — a key also names its instance (`sourceInstance`, the origin of
|
||||
* the Paperless-ngx or Papra it came from): id 123 on one Paperless and id
|
||||
* 123 on another are different documents. A null on either side is a
|
||||
* wildcard, so nothing stored before the column existed is forgotten:
|
||||
*
|
||||
* - a key stored without an instance (a v1.39.2 import, an older script, a
|
||||
* workflow that does not send one) matches a lookup for any instance of
|
||||
* its system;
|
||||
* - a lookup without an instance matches a key stored for any instance.
|
||||
*
|
||||
* Where both an exact and a wildcard match exist, the exact one wins.
|
||||
*/
|
||||
import { prisma } from "@/lib/db";
|
||||
import { checkRateLimit, type RateLimitResult } from "@/lib/rate-limit";
|
||||
@@ -23,22 +36,44 @@ export type SourceKeyMatch =
|
||||
| { state: "live"; document: SerialisableDocument }
|
||||
| { state: "deleted"; id: string | null };
|
||||
|
||||
/**
|
||||
* The instance half of a key match: the same instance or a stored key without
|
||||
* one; any instance when the lookup has none.
|
||||
*/
|
||||
export function instanceMatch(sourceInstance: string | null): {
|
||||
OR?: Array<{ sourceInstance: string | null }>;
|
||||
} {
|
||||
return sourceInstance === null
|
||||
? {}
|
||||
: { OR: [{ sourceInstance }, { sourceInstance: null }] };
|
||||
}
|
||||
|
||||
/** Exact instance first, then a key stored without one (nulls sort last). */
|
||||
const EXACT_FIRST = { sourceInstance: { sort: "asc", nulls: "last" } } as const;
|
||||
|
||||
export async function findSourceKey(
|
||||
userId: string,
|
||||
sourceSystem: DocumentSourceSystemValue,
|
||||
sourceId: string,
|
||||
sourceInstance: string | null = null,
|
||||
): Promise<SourceKeyMatch | null> {
|
||||
const key = {
|
||||
userId,
|
||||
sourceSystem,
|
||||
sourceId,
|
||||
...instanceMatch(sourceInstance),
|
||||
};
|
||||
const own = await prisma.inboundDocument.findFirst({
|
||||
where: { userId, sourceSystem, sourceId },
|
||||
where: key,
|
||||
omit: { contentEncrypted: true },
|
||||
orderBy: EXACT_FIRST,
|
||||
});
|
||||
const viaAlias = own
|
||||
? null
|
||||
: await prisma.documentSourceAlias.findUnique({
|
||||
where: {
|
||||
userId_sourceSystem_sourceId: { userId, sourceSystem, sourceId },
|
||||
},
|
||||
: await prisma.documentSourceAlias.findFirst({
|
||||
where: key,
|
||||
select: { document: { omit: { contentEncrypted: true } } },
|
||||
orderBy: EXACT_FIRST,
|
||||
});
|
||||
const document = own ?? viaAlias?.document ?? null;
|
||||
if (document) {
|
||||
@@ -46,10 +81,8 @@ export async function findSourceKey(
|
||||
? { state: "deleted", id: document.id }
|
||||
: { state: "live", document };
|
||||
}
|
||||
const purged = await prisma.documentImportKey.findUnique({
|
||||
where: {
|
||||
userId_sourceSystem_sourceId: { userId, sourceSystem, sourceId },
|
||||
},
|
||||
const purged = await prisma.documentImportKey.findFirst({
|
||||
where: key,
|
||||
select: { id: true },
|
||||
});
|
||||
return purged ? { state: "deleted", id: null } : null;
|
||||
@@ -74,13 +107,14 @@ export async function rememberSourceAlias(
|
||||
documentId: string,
|
||||
sourceSystem: DocumentSourceSystemValue,
|
||||
sourceId: string,
|
||||
sourceInstance: string | null = null,
|
||||
): Promise<"remembered" | "known" | "limit"> {
|
||||
const held = await prisma.documentSourceAlias.count({
|
||||
where: { userId, documentId },
|
||||
});
|
||||
if (held >= MAX_SOURCE_ALIASES_PER_DOCUMENT) return "limit";
|
||||
const { count } = await prisma.documentSourceAlias.createMany({
|
||||
data: [{ userId, documentId, sourceSystem, sourceId }],
|
||||
data: [{ userId, documentId, sourceSystem, sourceInstance, sourceId }],
|
||||
skipDuplicates: true,
|
||||
});
|
||||
return count > 0 ? "remembered" : "known";
|
||||
@@ -123,33 +157,56 @@ export async function findSourceKeyStates(
|
||||
userId: string,
|
||||
sourceSystem: DocumentSourceSystemValue,
|
||||
sourceIds: string[],
|
||||
sourceInstance: string | null = null,
|
||||
): Promise<Map<string, SourceKeyState>> {
|
||||
const states = new Map<string, SourceKeyState>();
|
||||
const ids = [...new Set(sourceIds)];
|
||||
if (ids.length === 0) return states;
|
||||
|
||||
const where = {
|
||||
userId,
|
||||
sourceSystem,
|
||||
sourceId: { in: ids },
|
||||
...instanceMatch(sourceInstance),
|
||||
};
|
||||
const [own, aliases, purged] = await Promise.all([
|
||||
prisma.inboundDocument.findMany({
|
||||
where: { userId, sourceSystem, sourceId: { in: ids } },
|
||||
select: { id: true, sourceId: true, deletedAt: true },
|
||||
where,
|
||||
select: {
|
||||
id: true,
|
||||
sourceId: true,
|
||||
sourceInstance: true,
|
||||
deletedAt: true,
|
||||
},
|
||||
}),
|
||||
prisma.documentSourceAlias.findMany({
|
||||
where: { userId, sourceSystem, sourceId: { in: ids } },
|
||||
where,
|
||||
select: {
|
||||
sourceId: true,
|
||||
sourceInstance: true,
|
||||
document: { select: { id: true, deletedAt: true } },
|
||||
},
|
||||
}),
|
||||
prisma.documentImportKey.findMany({
|
||||
where: { userId, sourceSystem, sourceId: { in: ids } },
|
||||
where,
|
||||
select: { sourceId: true },
|
||||
}),
|
||||
]);
|
||||
// A wildcard (instance-less) row first, so an exact-instance row after it
|
||||
// wins, as it does in `findSourceKey`.
|
||||
const wildcardFirst = <T extends { sourceInstance: string | null }>(
|
||||
rows: T[],
|
||||
) =>
|
||||
[...rows].sort(
|
||||
(a, b) =>
|
||||
Number(a.sourceInstance === sourceInstance) -
|
||||
Number(b.sourceInstance === sourceInstance),
|
||||
);
|
||||
|
||||
// Lowest precedence first, so the document's own key wins, as it does in
|
||||
// `findSourceKey`.
|
||||
for (const row of purged) states.set(row.sourceId, { state: "deleted" });
|
||||
for (const row of aliases) {
|
||||
for (const row of wildcardFirst(aliases)) {
|
||||
states.set(
|
||||
row.sourceId,
|
||||
row.document.deletedAt
|
||||
@@ -157,7 +214,7 @@ export async function findSourceKeyStates(
|
||||
: { state: "imported", documentId: row.document.id },
|
||||
);
|
||||
}
|
||||
for (const row of own) {
|
||||
for (const row of wildcardFirst(own)) {
|
||||
if (!row.sourceId) continue;
|
||||
states.set(
|
||||
row.sourceId,
|
||||
|
||||
@@ -171,8 +171,10 @@ describe("Paperless-ngx", () => {
|
||||
const list = calledUrls().find((u) => u.pathname.endsWith("/documents/"));
|
||||
expect(list?.searchParams.get("title__icontains")).toBe("blood");
|
||||
expect(list?.searchParams.get("tags__id__all")).toBe("7");
|
||||
expect(list?.searchParams.get("created__date__gte")).toBe("2025-01-01");
|
||||
expect(list?.searchParams.get("created__date__lte")).toBe("2025-12-31");
|
||||
expect(list?.searchParams.get("created__gte")).toBe("2025-01-01");
|
||||
expect(list?.searchParams.get("created__lte")).toBe("2025-12-31");
|
||||
// API version 9 made `created` a date; the `__date` lookups are gone.
|
||||
expect(list?.searchParams.has("created__date__gte")).toBe(false);
|
||||
expect(list?.searchParams.get("page")).toBe("2");
|
||||
expect(list?.searchParams.get("page_size")).toBe("25");
|
||||
|
||||
@@ -311,6 +313,76 @@ describe("Paperless-ngx", () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe("ids never shape a request path", () => {
|
||||
it("refuses a Paperless id that is not a number, and dot segments, before dialling", async () => {
|
||||
const client = sourceClient(PAPERLESS);
|
||||
for (const id of ["..", ".", "41/../x", "abc", "41?x=1"]) {
|
||||
const err = await failure(client.document(id));
|
||||
expect((err as DocumentSourceError).code, id).toBe("notFound");
|
||||
expect(
|
||||
((await failure(client.download(id, 10))) as DocumentSourceError).code,
|
||||
).toBe("notFound");
|
||||
}
|
||||
expect(safeFetch).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("refuses a Papra id or organization id outside [A-Za-z0-9_-]", async () => {
|
||||
for (const id of ["..", "doc/1", "doc%2F1", "doc 1"]) {
|
||||
await failure(sourceClient(PAPRA).document(id));
|
||||
}
|
||||
expect(() =>
|
||||
sourceClient({ ...PAPRA, organizationId: "../admin" }),
|
||||
).toThrow(DocumentSourceError);
|
||||
expect(safeFetch).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("leaves a result with an unusable id out of the list", async () => {
|
||||
serve({
|
||||
"/paperless/api/tags/": () => json({ results: [], next: null }),
|
||||
"/paperless/api/documents/": () =>
|
||||
json({
|
||||
next: null,
|
||||
results: [
|
||||
{ id: "..", title: "Odd", created: "2025-01-01", tags: [] },
|
||||
{ id: 5, title: "Fine", created: "2025-01-01", tags: [] },
|
||||
],
|
||||
}),
|
||||
});
|
||||
const result = await sourceClient(PAPERLESS).search({
|
||||
q: "",
|
||||
tagId: null,
|
||||
from: null,
|
||||
to: null,
|
||||
page: 1,
|
||||
});
|
||||
expect(result.items.map((i) => i.sourceId)).toEqual(["5"]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("Paperless-ngx tags", () => {
|
||||
it("pages through tags beyond the first 250", async () => {
|
||||
serve({
|
||||
"/paperless/api/tags/": (url) => {
|
||||
const page = Number(url.searchParams.get("page"));
|
||||
const results = Array.from(
|
||||
{ length: page === 1 ? 250 : 3 },
|
||||
(_, i) => ({
|
||||
id: page * 1000 + i,
|
||||
name: `t${page}-${i}`,
|
||||
}),
|
||||
);
|
||||
return json({ results, next: page === 1 ? "more" : null });
|
||||
},
|
||||
});
|
||||
const tags = await sourceClient(PAPERLESS).tags();
|
||||
expect(tags).toHaveLength(253);
|
||||
expect(calledUrls().map((u) => u.searchParams.get("page"))).toEqual([
|
||||
"1",
|
||||
"2",
|
||||
]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("Papra", () => {
|
||||
const tags = {
|
||||
tags: [
|
||||
|
||||
@@ -29,6 +29,8 @@ import {
|
||||
import {
|
||||
DOCUMENT_PICKER_MAX_TAGS,
|
||||
DOCUMENT_PICKER_PAGE_SIZE,
|
||||
isSourceDocumentId,
|
||||
PAPRA_ID,
|
||||
type DocumentPickerSystem,
|
||||
type DocumentSourceTagDto,
|
||||
} from "./types";
|
||||
@@ -64,6 +66,11 @@ export interface SourceDocumentMeta {
|
||||
|
||||
export interface SourceClient {
|
||||
readonly system: DocumentPickerSystem;
|
||||
/**
|
||||
* The instance this client talks to, as the normalised origin every source
|
||||
* key it produces carries (`sourceInstance`).
|
||||
*/
|
||||
readonly instance: string;
|
||||
test(): Promise<void>;
|
||||
tags(): Promise<DocumentSourceTagDto[]>;
|
||||
search(
|
||||
@@ -84,6 +91,16 @@ export interface SourceConnection {
|
||||
token: string;
|
||||
}
|
||||
|
||||
/** Paperless tags are read this many per page. */
|
||||
const TAG_PAGE_SIZE = 250;
|
||||
|
||||
/** Refuse an id that is not one the system issues, before it reaches a URL. */
|
||||
function requireId(system: DocumentPickerSystem, id: string): void {
|
||||
if (!isSourceDocumentId(system, id)) {
|
||||
throw new DocumentSourceError("notFound");
|
||||
}
|
||||
}
|
||||
|
||||
// ── Defensive readers ──────────────────────────────────────────────────────
|
||||
|
||||
type Json = Record<string, unknown>;
|
||||
@@ -181,26 +198,36 @@ function paperless(connection: SourceConnection): SourceClient {
|
||||
}
|
||||
|
||||
async function listTags(): Promise<DocumentSourceTagDto[]> {
|
||||
const query = new URLSearchParams({
|
||||
page_size: String(DOCUMENT_PICKER_MAX_TAGS),
|
||||
ordering: "name",
|
||||
});
|
||||
const { body, response } = await sourceJson(target, `/api/tags/?${query}`);
|
||||
checkVersion(response);
|
||||
const page = asRecord(body);
|
||||
if (!page) throw shapeError();
|
||||
// Paged by number rather than by following `next`: the link is the
|
||||
// server's to write, the page number is ours.
|
||||
const tags: DocumentSourceTagDto[] = [];
|
||||
for (const raw of asArray(page.results)) {
|
||||
const tag = asRecord(raw);
|
||||
const id = asText(tag?.id);
|
||||
const name = asText(tag?.name);
|
||||
if (id && name) tags.push({ id, name });
|
||||
for (let pageNo = 1; tags.length < DOCUMENT_PICKER_MAX_TAGS; pageNo += 1) {
|
||||
const query = new URLSearchParams({
|
||||
page: String(pageNo),
|
||||
page_size: String(TAG_PAGE_SIZE),
|
||||
ordering: "name",
|
||||
});
|
||||
const { body, response } = await sourceJson(
|
||||
target,
|
||||
`/api/tags/?${query}`,
|
||||
);
|
||||
checkVersion(response);
|
||||
const page = asRecord(body);
|
||||
if (!page) throw shapeError();
|
||||
for (const raw of asArray(page.results)) {
|
||||
const tag = asRecord(raw);
|
||||
const id = asText(tag?.id);
|
||||
const name = asText(tag?.name);
|
||||
if (id && name) tags.push({ id, name });
|
||||
}
|
||||
if (typeof page.next !== "string" || !page.next) break;
|
||||
}
|
||||
return tags;
|
||||
return tags.slice(0, DOCUMENT_PICKER_MAX_TAGS);
|
||||
}
|
||||
|
||||
return {
|
||||
system: "PAPERLESS",
|
||||
instance: connection.origin,
|
||||
|
||||
async test() {
|
||||
const { body, response } = await sourceJson(
|
||||
@@ -223,8 +250,10 @@ function paperless(connection: SourceConnection): SourceClient {
|
||||
});
|
||||
if (params.q) query.set("title__icontains", params.q);
|
||||
if (params.tagId) query.set("tags__id__all", params.tagId);
|
||||
if (params.from) query.set("created__date__gte", params.from);
|
||||
if (params.to) query.set("created__date__lte", params.to);
|
||||
// API version 9 made `created` a plain date, so the range filters on
|
||||
// it directly (`created__gte` / `created__lte`), both ends inclusive.
|
||||
if (params.from) query.set("created__gte", params.from);
|
||||
if (params.to) query.set("created__lte", params.to);
|
||||
|
||||
let page: Json | null;
|
||||
let names: Map<string, string>;
|
||||
@@ -250,7 +279,7 @@ function paperless(connection: SourceConnection): SourceClient {
|
||||
for (const raw of asArray(page.results)) {
|
||||
const doc = asRecord(raw);
|
||||
const id = asText(doc?.id);
|
||||
if (!doc || !id) continue;
|
||||
if (!doc || !id || !isSourceDocumentId("PAPERLESS", id)) continue;
|
||||
const tagIds = asArray(doc.tags)
|
||||
.map(asText)
|
||||
.filter((t): t is string => t !== null);
|
||||
@@ -275,6 +304,7 @@ function paperless(connection: SourceConnection): SourceClient {
|
||||
},
|
||||
|
||||
async document(sourceId) {
|
||||
requireId("PAPERLESS", sourceId);
|
||||
const { body, response } = await sourceJson(
|
||||
target,
|
||||
`/api/documents/${encodeURIComponent(sourceId)}/?fields=id,title,created,added,original_file_name`,
|
||||
@@ -290,7 +320,8 @@ function paperless(connection: SourceConnection): SourceClient {
|
||||
};
|
||||
},
|
||||
|
||||
download(sourceId, maxFileBytes) {
|
||||
async download(sourceId, maxFileBytes) {
|
||||
requireId("PAPERLESS", sourceId);
|
||||
// The original, not the archived PDF/A: the same bytes a Paperless
|
||||
// workflow or the import script sends, so they dedup against each other.
|
||||
return sourceDownload(
|
||||
@@ -310,7 +341,7 @@ function papraTagTerm(name: string): string {
|
||||
}
|
||||
|
||||
function papra(connection: SourceConnection): SourceClient {
|
||||
if (!connection.organizationId) {
|
||||
if (!connection.organizationId || !PAPRA_ID.test(connection.organizationId)) {
|
||||
throw new DocumentSourceError("notFound");
|
||||
}
|
||||
const org = `/api/organizations/${encodeURIComponent(connection.organizationId)}`;
|
||||
@@ -345,6 +376,7 @@ function papra(connection: SourceConnection): SourceClient {
|
||||
|
||||
return {
|
||||
system: "PAPRA",
|
||||
instance: connection.origin,
|
||||
|
||||
async test() {
|
||||
const { body } = await sourceJson(
|
||||
@@ -381,7 +413,7 @@ function papra(connection: SourceConnection): SourceClient {
|
||||
for (const raw of documents) {
|
||||
const doc = asRecord(raw);
|
||||
const id = asText(doc?.id);
|
||||
if (!doc || !id) continue;
|
||||
if (!doc || !id || !isSourceDocumentId("PAPRA", id)) continue;
|
||||
const tags = asArray(doc.tags)
|
||||
.map(asRecord)
|
||||
.filter((t): t is Json => t !== null);
|
||||
@@ -417,6 +449,7 @@ function papra(connection: SourceConnection): SourceClient {
|
||||
},
|
||||
|
||||
async document(sourceId) {
|
||||
requireId("PAPRA", sourceId);
|
||||
const { body } = await sourceJson(
|
||||
target,
|
||||
`${org}/documents/${encodeURIComponent(sourceId)}`,
|
||||
@@ -435,7 +468,8 @@ function papra(connection: SourceConnection): SourceClient {
|
||||
};
|
||||
},
|
||||
|
||||
download(sourceId, maxFileBytes) {
|
||||
async download(sourceId, maxFileBytes) {
|
||||
requireId("PAPRA", sourceId);
|
||||
return sourceDownload(
|
||||
download,
|
||||
`${org}/documents/${encodeURIComponent(sourceId)}/file`,
|
||||
|
||||
@@ -39,12 +39,13 @@ import {
|
||||
* Each route calls `requireAuth()` itself, in its own module, so the sharing
|
||||
* guard's scan sees every one of them as a bare, owner-only route rather than
|
||||
* a helper it cannot follow. This adds the rest: a cookie transport, the
|
||||
* documents module, and (unless `needsList` is false, which only the status
|
||||
* read passes) the operator's list.
|
||||
* documents module and the operator's list. The status read and the delete
|
||||
* pass `needsList: false, needsModule: false`: they touch only the person's
|
||||
* own row and never the network.
|
||||
*/
|
||||
export async function admitDocumentSourceCaller(
|
||||
auth: AuthContext,
|
||||
options: { needsList?: boolean } = {},
|
||||
options: { needsList?: boolean; needsModule?: boolean } = {},
|
||||
): Promise<Response | null> {
|
||||
if (auth.authMethod !== "cookie") {
|
||||
return apiError(
|
||||
@@ -53,8 +54,13 @@ export async function admitDocumentSourceCaller(
|
||||
{ errorCode: "documents.sources.browserOnly" },
|
||||
);
|
||||
}
|
||||
const gate = await requireModuleEnabled(auth.user.id, "inboundDocuments");
|
||||
if (!gate.enabled) return gate.response;
|
||||
// Reading and removing one's own connection needs neither the module nor
|
||||
// the list: a person who switched Documents off, or whose operator removed
|
||||
// the list, must still be able to see and delete the stored token.
|
||||
if (options.needsModule !== false) {
|
||||
const gate = await requireModuleEnabled(auth.user.id, "inboundDocuments");
|
||||
if (!gate.enabled) return gate.response;
|
||||
}
|
||||
if (options.needsList !== false && !documentSourcesAvailable()) {
|
||||
return sourceErrorResponse("unavailable");
|
||||
}
|
||||
|
||||
@@ -38,8 +38,26 @@ export const DOCUMENT_PICKER_MAX_PAGE = 500;
|
||||
export const DOCUMENT_PICKER_QUERY_MAX = 200;
|
||||
/** Documents the sheet imports in one run. */
|
||||
export const DOCUMENT_PICKER_MAX_SELECTION = 25;
|
||||
/** Tags offered in the filter. */
|
||||
export const DOCUMENT_PICKER_MAX_TAGS = 250;
|
||||
/** Tags offered in the filter (read in pages of 250). */
|
||||
export const DOCUMENT_PICKER_MAX_TAGS = 1000;
|
||||
|
||||
/** Paperless-ngx document ids are integers. */
|
||||
export const PAPERLESS_ID = /^\d{1,18}$/;
|
||||
/** Papra document and organization ids: letters, digits, `_` and `-`. */
|
||||
export const PAPRA_ID = /^[A-Za-z0-9_-]{1,128}$/;
|
||||
|
||||
/**
|
||||
* Is `id` a document id `system` could have issued? Checked before an id is
|
||||
* put into a request path, so nothing a request carries can add a path
|
||||
* segment, a query or a `..` to the URL built from it.
|
||||
*/
|
||||
export function isSourceDocumentId(
|
||||
system: DocumentPickerSystem,
|
||||
id: string,
|
||||
): boolean {
|
||||
if (id === "." || id === "..") return false;
|
||||
return (system === "PAPERLESS" ? PAPERLESS_ID : PAPRA_ID).test(id);
|
||||
}
|
||||
|
||||
/** A connection as every route may show it: never the token. */
|
||||
export interface DocumentSourceConnectionDto {
|
||||
|
||||
@@ -177,6 +177,8 @@ export interface DocumentBackupEntry {
|
||||
*/
|
||||
sourceSystem?: string | null;
|
||||
sourceId?: string | null;
|
||||
/** v1.39.3 — the source instance (origin); absent in older files. */
|
||||
sourceInstance?: string | null;
|
||||
/** Canonical DR only: the import's hold on automatic AI reading. */
|
||||
aiReadDeferred?: boolean;
|
||||
}
|
||||
@@ -309,6 +311,7 @@ export async function buildRecordsBackupSection(
|
||||
lastIndexOutcome: true,
|
||||
sourceSystem: true,
|
||||
sourceId: true,
|
||||
sourceInstance: true,
|
||||
aiReadDeferred: true,
|
||||
createdAt: true,
|
||||
updatedAt: true,
|
||||
@@ -330,6 +333,7 @@ export async function buildRecordsBackupSection(
|
||||
summaryEncrypted: true,
|
||||
sourceSystem: true,
|
||||
sourceId: true,
|
||||
sourceInstance: true,
|
||||
createdAt: true,
|
||||
},
|
||||
});
|
||||
@@ -545,6 +549,7 @@ export async function buildRecordsBackupSection(
|
||||
: null,
|
||||
sourceSystem: d.sourceSystem,
|
||||
sourceId: d.sourceId,
|
||||
sourceInstance: d.sourceInstance,
|
||||
createdAt: d.createdAt.toISOString(),
|
||||
};
|
||||
|
||||
|
||||
@@ -30,6 +30,7 @@
|
||||
import { Buffer } from "node:buffer";
|
||||
|
||||
import { prisma, toJson } from "@/lib/db";
|
||||
import { normaliseSourceInstance } from "@/lib/validations/inbound-documents";
|
||||
import { auditLog } from "@/lib/auth/audit";
|
||||
import {
|
||||
isStoredBackupReadError,
|
||||
@@ -2057,6 +2058,14 @@ export async function restoreBackup(
|
||||
sourceId: document.sourceSystem
|
||||
? (document.sourceId ?? null)
|
||||
: null,
|
||||
// A file written before v1.39.3 has no instance: the key then
|
||||
// matches any instance of its system, as it did when stored.
|
||||
sourceInstance:
|
||||
document.sourceSystem && document.sourceId
|
||||
? document.sourceInstance
|
||||
? normaliseSourceInstance(document.sourceInstance)
|
||||
: null
|
||||
: null,
|
||||
createdAt: new Date(document.createdAt!),
|
||||
updatedAt: new Date(document.updatedAt!),
|
||||
})),
|
||||
|
||||
@@ -58,9 +58,12 @@ export async function purgeExpiredDocumentTombstones(
|
||||
userId: true,
|
||||
sourceSystem: true,
|
||||
sourceId: true,
|
||||
sourceInstance: true,
|
||||
// Further keys the import answered with this document (same bytes).
|
||||
// They cascade away with the row, so they are copied here first.
|
||||
sourceAliases: { select: { sourceSystem: true, sourceId: true } },
|
||||
sourceAliases: {
|
||||
select: { sourceSystem: true, sourceId: true, sourceInstance: true },
|
||||
},
|
||||
},
|
||||
});
|
||||
const keys = keyed.flatMap((row) => [
|
||||
@@ -70,6 +73,7 @@ export async function purgeExpiredDocumentTombstones(
|
||||
userId: row.userId,
|
||||
sourceSystem: row.sourceSystem,
|
||||
sourceId: row.sourceId,
|
||||
sourceInstance: row.sourceInstance,
|
||||
},
|
||||
]
|
||||
: []),
|
||||
@@ -77,6 +81,7 @@ export async function purgeExpiredDocumentTombstones(
|
||||
userId: row.userId,
|
||||
sourceSystem: alias.sourceSystem,
|
||||
sourceId: alias.sourceId,
|
||||
sourceInstance: alias.sourceInstance,
|
||||
})),
|
||||
]);
|
||||
if (keys.length > 0) {
|
||||
|
||||
@@ -83,6 +83,7 @@ import {
|
||||
} from "@/lib/validations/emergency-profile";
|
||||
import {
|
||||
DOCUMENT_SOURCE_ID_MAX,
|
||||
DOCUMENT_SOURCE_INSTANCE_MAX,
|
||||
DOCUMENT_SOURCE_SYSTEMS,
|
||||
} from "@/lib/validations/inbound-documents";
|
||||
import { REMINDER_EVENT_SOURCES } from "@/lib/measurement-reminders/satisfy";
|
||||
@@ -1636,6 +1637,14 @@ const documentBackupSchema = z
|
||||
// restore cannot write a system the detail sheet has no name for.
|
||||
sourceSystem: z.enum(DOCUMENT_SOURCE_SYSTEMS).nullable().optional(),
|
||||
sourceId: z.string().max(DOCUMENT_SOURCE_ID_MAX).nullable().optional(),
|
||||
// v1.39.3 — the source instance (an origin). Length-bounded here and
|
||||
// normalised on restore; a value that is not an origin restores as null
|
||||
// (matches any instance) rather than failing the restore.
|
||||
sourceInstance: z
|
||||
.string()
|
||||
.max(DOCUMENT_SOURCE_INSTANCE_MAX)
|
||||
.nullable()
|
||||
.optional(),
|
||||
aiReadDeferred: z.boolean().optional(),
|
||||
summary: z.string().nullable().optional(),
|
||||
createdAt: isoDateTime.optional(),
|
||||
|
||||
@@ -10,6 +10,7 @@ import {
|
||||
DOCUMENT_PICKER_LINK_KINDS,
|
||||
DOCUMENT_PICKER_MAX_PAGE,
|
||||
DOCUMENT_PICKER_QUERY_MAX,
|
||||
PAPRA_ID,
|
||||
} from "@/lib/documents/sources/types";
|
||||
import {
|
||||
documentSourceIdSchema,
|
||||
@@ -37,9 +38,7 @@ export const documentSourceSaveSchema = z.object({
|
||||
organizationId: z
|
||||
.string()
|
||||
.trim()
|
||||
.min(1)
|
||||
.max(128)
|
||||
.regex(printable, "Expected printable characters without spaces")
|
||||
.regex(PAPRA_ID, "Expected letters, digits, _ or -")
|
||||
.optional(),
|
||||
token: z
|
||||
.string()
|
||||
|
||||
@@ -706,7 +706,41 @@ export const documentSourceIdSchema = z
|
||||
.trim()
|
||||
.min(1)
|
||||
.max(DOCUMENT_SOURCE_ID_MAX)
|
||||
.regex(/^[\x21-\x7e]+$/u, "Expected printable characters without spaces");
|
||||
.regex(/^[\x21-\x7e]+$/u, "Expected printable characters without spaces")
|
||||
// A path segment of its own name: an importer that builds a URL from the id
|
||||
// must never be handed one that walks up the source's path.
|
||||
.refine((v) => v !== "." && v !== "..", "Not a document id");
|
||||
|
||||
/** Longest source instance (an origin) a key may carry. */
|
||||
export const DOCUMENT_SOURCE_INSTANCE_MAX = 512;
|
||||
|
||||
/**
|
||||
* v1.39.3 — the instance of the source system a key belongs to, as the
|
||||
* normalised origin (`scheme://host[:port]`) of the Paperless-ngx or Papra it
|
||||
* came from, or null when the value is not one. A path, query or credentials
|
||||
* in the value are dropped: two addresses on one origin are one instance.
|
||||
*/
|
||||
export function normaliseSourceInstance(value: string): string | null {
|
||||
try {
|
||||
const url = new URL(value.trim());
|
||||
if (url.protocol !== "http:" && url.protocol !== "https:") return null;
|
||||
if (url.username || url.password) return null;
|
||||
return url.origin;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/** The instance as a request field: validated, then normalised to its origin. */
|
||||
export const documentSourceInstanceSchema = z
|
||||
.string()
|
||||
.trim()
|
||||
.min(1)
|
||||
.max(DOCUMENT_SOURCE_INSTANCE_MAX)
|
||||
.refine((v) => normaliseSourceInstance(v) !== null, {
|
||||
message: "Expected the http(s) address of the source instance",
|
||||
})
|
||||
.transform((v) => normaliseSourceInstance(v) as string);
|
||||
|
||||
/**
|
||||
* A source key on its own — the query-string form the upload accepts ahead of
|
||||
@@ -715,6 +749,8 @@ export const documentSourceIdSchema = z
|
||||
export const documentSourceKeySchema = z.object({
|
||||
sourceSystem: z.enum(DOCUMENT_SOURCE_SYSTEMS),
|
||||
sourceId: documentSourceIdSchema,
|
||||
/** Optional: without it the key matches any instance of the system. */
|
||||
sourceInstance: documentSourceInstanceSchema.optional(),
|
||||
});
|
||||
|
||||
export const documentCreateSchema = z
|
||||
@@ -731,6 +767,11 @@ export const documentCreateSchema = z
|
||||
* not prose, and it is echoed on the detail sheet.
|
||||
*/
|
||||
sourceId: documentSourceIdSchema.optional(),
|
||||
/**
|
||||
* v1.39.3 — which instance of `sourceSystem` (its origin). Optional; a
|
||||
* key without one matches any instance.
|
||||
*/
|
||||
sourceInstance: documentSourceInstanceSchema.optional(),
|
||||
/**
|
||||
* `defer` holds back automatic AI reading for this upload: the thumbnail
|
||||
* and a local text index still run, the summary and the lab staging do
|
||||
@@ -741,6 +782,10 @@ export const documentCreateSchema = z
|
||||
.refine((v) => v.sourceId === undefined || v.sourceSystem !== undefined, {
|
||||
path: ["sourceSystem"],
|
||||
message: "sourceId needs a sourceSystem",
|
||||
})
|
||||
.refine((v) => v.sourceInstance === undefined || v.sourceId !== undefined, {
|
||||
path: ["sourceInstance"],
|
||||
message: "sourceInstance needs a sourceId",
|
||||
});
|
||||
|
||||
export type DocumentCreateInput = z.infer<typeof documentCreateSchema>;
|
||||
|
||||
Reference in new issue
Block a user