mirror of
https://github.com/MBombeck/HealthLog.git
synced 2026-11-14 22:50:51 +00:00
Merge branch 'feat/document-picker' into release/v1.39.3
This commit is contained in:
Marc-André Bombeck committed
commit
a8cf4c1dad
71 files changed
+8934
-375
No files matched your search
@@ -161,6 +161,14 @@ export const ENCRYPTED_COLUMNS: readonly EncryptedColumn[] = [
|
||||
{ model: "GoogleHealthConnection", field: "accessToken", kind: "string" },
|
||||
{ model: "GoogleHealthConnection", field: "refreshToken", kind: "string" },
|
||||
|
||||
// ───── Document picker (v1.39.3, #1038) ─────
|
||||
// A person's API token for their Paperless-ngx or Papra instance.
|
||||
{
|
||||
model: "DocumentSourceConnection",
|
||||
field: "tokenEncrypted",
|
||||
kind: "string",
|
||||
},
|
||||
|
||||
// ───── AppSettings — operator credentials ─────
|
||||
{ model: "AppSettings", field: "adminAiKeyEncrypted", kind: "string" },
|
||||
{
|
||||
|
||||
@@ -155,6 +155,10 @@ export const WIPE_MODELS = [
|
||||
// Further import keys that resolved to a document by content; a child of
|
||||
// the document, removed before it so the counts are truthful.
|
||||
"DocumentSourceAlias",
|
||||
// The person's own connections to Paperless-ngx / Papra (#1038), with their
|
||||
// encrypted API tokens. A wipe that left them would leave a working
|
||||
// credential to another system behind an account that erased everything.
|
||||
"DocumentSourceConnection",
|
||||
"ExtractedFact",
|
||||
"ImportJob",
|
||||
|
||||
|
||||
@@ -0,0 +1,435 @@
|
||||
/**
|
||||
* One way into the vault for a file that has already been read into memory.
|
||||
*
|
||||
* The upload route (`POST /api/documents/inbound`, cookie, `["*"]` or a
|
||||
* `documents:write` token) and the document picker's import route
|
||||
* (`POST /api/documents/sources/{system}/import`, which fetches a picked
|
||||
* document from Paperless-ngx or Papra) both store through here, so the rules a
|
||||
* stored document is held to cannot differ by the way it arrived:
|
||||
*
|
||||
* - a source key answers before the bytes are looked at: a key already held
|
||||
* (live, tombstoned, remembered as an alias or kept in the purge ledger) is
|
||||
* a duplicate or a "deleted", never a second copy;
|
||||
* - the per-file cap, the empty-file refusal and magic-byte classification
|
||||
* (the wire Content-Type is never trusted);
|
||||
* - pre-links to the caller's own live episodes and visits, refused when an
|
||||
* id names nothing the caller owns;
|
||||
* - sha256 duplicate detection against live rows, remembering the source key
|
||||
* for the existing document so a later delete keeps that key deleted too;
|
||||
* for an import, also against deleted (not yet purged) rows, which answer
|
||||
* "deleted" rather than storing the file again;
|
||||
* - the quota gate and the insert in one transaction under a per-user
|
||||
* advisory lock, and the race on either partial unique index resolved the
|
||||
* way the fast paths above would have answered;
|
||||
* - the audit row, the wide-event annotation, and the background index,
|
||||
* thumbnail and summary jobs (the summary only when the `documentAi`
|
||||
* capability is open and the upload did not ask to defer AI reading).
|
||||
*
|
||||
* The callers own authentication, rate buckets, request parsing and the shape
|
||||
* of their responses. This module returns what happened; it builds no HTTP.
|
||||
*/
|
||||
import type { Buffer } from "node:buffer";
|
||||
import { createHash } from "node:crypto";
|
||||
|
||||
import type { AiCapabilityState } from "@/lib/ai/capabilities/types";
|
||||
import { auditLog } from "@/lib/auth/audit";
|
||||
import { prisma } from "@/lib/db";
|
||||
import {
|
||||
narrowOwnedEncounterIds,
|
||||
narrowOwnedEpisodeIds,
|
||||
} from "@/lib/documents/links";
|
||||
import {
|
||||
findSourceKey,
|
||||
rememberSourceAlias,
|
||||
type SourceKeyMatch,
|
||||
} from "@/lib/documents/source-key";
|
||||
import {
|
||||
encryptDocumentContent,
|
||||
type SerialisableDocument,
|
||||
} from "@/lib/documents/store";
|
||||
import { detectDocumentType } from "@/lib/documents/upload-policy";
|
||||
import { enqueueDocumentIndex } from "@/lib/jobs/document-index";
|
||||
import { enqueueDocumentSummary } from "@/lib/jobs/document-summary";
|
||||
import { enqueueDocumentThumbnail } from "@/lib/jobs/document-thumbnail";
|
||||
import { linkTargets } from "@/lib/links";
|
||||
import { annotate } from "@/lib/logging/context";
|
||||
import { isP2002 } from "@/lib/prisma-errors";
|
||||
import type {
|
||||
DocumentSourceSystemValue,
|
||||
InboundDocumentKindValue,
|
||||
} from "@/lib/validations/inbound-documents";
|
||||
|
||||
/**
|
||||
* The person's own uploads per hour, from the web or the phone, and the
|
||||
* picker's imports with them. A store-only upload touches no provider — the
|
||||
* only abuse vector is disk, so a generous ceiling is enough. A narrow
|
||||
* `documents:write` token draws on its own bucket instead
|
||||
* (`resolveDocumentUploadLimitPerHour`).
|
||||
*/
|
||||
export const UPLOAD_LIMIT_PER_HOUR = 60;
|
||||
export const UPLOAD_WINDOW_MS = 60 * 60 * 1000;
|
||||
|
||||
/** The bucket a person's own uploads and picker imports share. */
|
||||
export function personalUploadBucket(userId: string): string {
|
||||
return `documents-upload:${userId}`;
|
||||
}
|
||||
|
||||
/** Parse a YYYY-MM-DD value into a UTC midnight Date. */
|
||||
export function isoDateToUtc(value: string): Date {
|
||||
return new Date(`${value}T00:00:00.000Z`);
|
||||
}
|
||||
|
||||
export interface IngestInput {
|
||||
userId: string;
|
||||
/** A narrow `documents:write` token rather than a session or `["*"]`. */
|
||||
scoped: boolean;
|
||||
/** For the audit row. */
|
||||
ipAddress: string | null;
|
||||
bytes: Buffer;
|
||||
filename: string | null;
|
||||
title: string | null;
|
||||
kind: InboundDocumentKindValue | null;
|
||||
/** YYYY-MM-DD; the upload day when absent. */
|
||||
documentDate: string | null;
|
||||
episodeIds: string[];
|
||||
encounterIds: string[];
|
||||
sourceSystem: DocumentSourceSystemValue | null;
|
||||
sourceId: string | null;
|
||||
/**
|
||||
* The source instance (origin) of the key, or null for a key sent without
|
||||
* one; see `src/lib/documents/source-key.ts` for how null matches.
|
||||
*/
|
||||
sourceInstance: string | null;
|
||||
/** Hold back automatic AI reading (`aiRead=defer`). */
|
||||
aiDeferred: boolean;
|
||||
/**
|
||||
* The caller already asked about this source key (the upload's query-string
|
||||
* form answers before the bucket is charged). Skips the second lookup.
|
||||
*/
|
||||
sourceKeyChecked: boolean;
|
||||
/** The per-file cap and the quota, resolved by the caller. */
|
||||
limits: { maxFileBytes: number; quotaBytes: number };
|
||||
/**
|
||||
* The `documentAi` capability, asked by the caller's route only once a new
|
||||
* document was stored (a duplicate never asks). Passed in rather than asked
|
||||
* here so each route that can queue AI work names the capability in its own
|
||||
* source, where the AI route inventory reads it.
|
||||
*/
|
||||
documentAi: () => Promise<AiCapabilityState>;
|
||||
}
|
||||
|
||||
export type IngestResult =
|
||||
| { kind: "sourceKey"; match: SourceKeyMatch }
|
||||
| { kind: "tooLarge"; maxFileBytes: number }
|
||||
| { kind: "empty" }
|
||||
| { kind: "unsupportedType" }
|
||||
| { kind: "episodeNotFound" }
|
||||
| { kind: "encounterNotFound" }
|
||||
| { kind: "aliasLimit" }
|
||||
| { kind: "duplicate"; document: SerialisableDocument }
|
||||
| { kind: "quotaExceeded"; usedBytes: number; quotaBytes: number }
|
||||
| {
|
||||
kind: "stored";
|
||||
document: SerialisableDocument;
|
||||
servingClass: string;
|
||||
linkedEpisodes: number;
|
||||
linkedVisits: number;
|
||||
};
|
||||
|
||||
/** Internal signal: the quota gate inside the insert transaction tripped. */
|
||||
class QuotaExceededError extends Error {
|
||||
constructor(public readonly usedBytes: number) {
|
||||
super("Document quota exceeded");
|
||||
this.name = "QuotaExceededError";
|
||||
}
|
||||
}
|
||||
|
||||
/** A live row with the same bytes, remembering the source key for it. */
|
||||
async function answerDuplicate(
|
||||
input: IngestInput,
|
||||
existing: SerialisableDocument,
|
||||
): Promise<IngestResult> {
|
||||
// An import sending bytes that are already stored under another key (or
|
||||
// none) is answered with that document — and the key is remembered for it,
|
||||
// so once the person deletes the document this key stays deleted too.
|
||||
if (input.sourceSystem && input.sourceId) {
|
||||
const remembered = await rememberSourceAlias(
|
||||
input.userId,
|
||||
existing.id,
|
||||
input.sourceSystem,
|
||||
input.sourceId,
|
||||
input.sourceInstance,
|
||||
);
|
||||
if (remembered === "limit") return { kind: "aliasLimit" };
|
||||
}
|
||||
return { kind: "duplicate", document: existing };
|
||||
}
|
||||
|
||||
export async function ingestDocument(
|
||||
input: IngestInput,
|
||||
): Promise<IngestResult> {
|
||||
const { userId, sourceSystem, sourceId, limits, bytes } = input;
|
||||
const sourceInstance = sourceId ? input.sourceInstance : null;
|
||||
|
||||
// A source key answers before the bytes are looked at: an import re-sending
|
||||
// what it sent before is a duplicate, and one re-sending a document the
|
||||
// person deleted is refused a second copy — live and tombstoned rows both
|
||||
// count (the unique index has no `deleted_at` predicate), and past the purge
|
||||
// the ledger remembers.
|
||||
if (sourceSystem && sourceId && !input.sourceKeyChecked) {
|
||||
const match = await findSourceKey(
|
||||
userId,
|
||||
sourceSystem,
|
||||
sourceId,
|
||||
sourceInstance,
|
||||
);
|
||||
if (match) return { kind: "sourceKey", match };
|
||||
}
|
||||
|
||||
if (bytes.byteLength > limits.maxFileBytes) {
|
||||
return { kind: "tooLarge", maxFileBytes: limits.maxFileBytes };
|
||||
}
|
||||
if (bytes.byteLength === 0) return { kind: "empty" };
|
||||
|
||||
// Magic-byte classification; the wire Content-Type is never trusted.
|
||||
const detected = detectDocumentType(bytes, input.filename);
|
||||
if (!detected) return { kind: "unsupportedType" };
|
||||
|
||||
// Pre-linking: every episode id must be a LIVE episode of the caller.
|
||||
const episodeIds = await narrowOwnedEpisodeIds(userId, input.episodeIds);
|
||||
if (episodeIds === null) return { kind: "episodeNotFound" };
|
||||
|
||||
// Same for the visit ids the review step offered. A refusal rather than a
|
||||
// silent drop: the person saw a visit named and would otherwise watch the
|
||||
// upload succeed without the link they asked for.
|
||||
const encounterIds = await narrowOwnedEncounterIds(
|
||||
userId,
|
||||
input.encounterIds,
|
||||
);
|
||||
if (encounterIds === null) return { kind: "encounterNotFound" };
|
||||
|
||||
// sha256 of the PLAINTEXT for same-user duplicate detection.
|
||||
const contentSha256 = createHash("sha256").update(bytes).digest("hex");
|
||||
|
||||
// Fast-path dedupe check; the partial unique index closes the race below.
|
||||
const existing = await prisma.inboundDocument.findFirst({
|
||||
where: { userId, contentSha256, deletedAt: null },
|
||||
omit: { contentEncrypted: true },
|
||||
});
|
||||
if (existing) return answerDuplicate(input, existing);
|
||||
|
||||
// An import (a source key is present) of bytes the person deleted here,
|
||||
// arriving under a key HealthLog has not seen: another system, another
|
||||
// instance, or a re-scan with a new id. The deletion was a decision about
|
||||
// the document, not about one key, so it is answered "deleted", and the
|
||||
// new key is remembered on the tombstone so the purge carries it into the
|
||||
// ledger. After the 30-day purge the bytes and their hash are gone and only
|
||||
// the keys remain; no content hash of a deleted document is kept. A person
|
||||
// uploading the file by hand (no key) is making a new copy on purpose.
|
||||
if (sourceSystem && sourceId) {
|
||||
const tombstone = await prisma.inboundDocument.findFirst({
|
||||
where: { userId, contentSha256, deletedAt: { not: null } },
|
||||
select: { id: true },
|
||||
orderBy: { deletedAt: "desc" },
|
||||
});
|
||||
if (tombstone) {
|
||||
const remembered = await rememberSourceAlias(
|
||||
userId,
|
||||
tombstone.id,
|
||||
sourceSystem,
|
||||
sourceId,
|
||||
sourceInstance,
|
||||
);
|
||||
if (remembered === "limit") return { kind: "aliasLimit" };
|
||||
return {
|
||||
kind: "sourceKey",
|
||||
match: { state: "deleted", id: tombstone.id },
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
// Quota gate + insert + pre-links in ONE transaction. Usage counts every
|
||||
// non-purged row — tombstones still hold TOAST bytes, so "deleted" bytes
|
||||
// are never invisible weight (undo-delete never changes usage).
|
||||
let document: SerialisableDocument;
|
||||
try {
|
||||
document = await prisma.$transaction(async (tx) => {
|
||||
// Serialise the quota gate per user: without this, N concurrent
|
||||
// uploads all read the same SUM before any of them commits and the
|
||||
// quota can be overshot by up to N × cap in one burst. The advisory
|
||||
// lock is transaction-scoped (released on commit/rollback) and keyed
|
||||
// on the user id, so uploads by different users never queue on each
|
||||
// other.
|
||||
// (`pg_advisory_xact_lock` returns void, which the client cannot
|
||||
// deserialize as a column — selecting FROM it yields a plain int row.)
|
||||
await tx.$queryRaw`
|
||||
SELECT 1 AS locked
|
||||
FROM pg_advisory_xact_lock(hashtextextended('documents-quota:' || ${userId}, 0))
|
||||
`;
|
||||
const rows = await tx.$queryRaw<Array<{ used: bigint }>>`
|
||||
SELECT COALESCE(SUM(byte_size), 0)::bigint AS used
|
||||
FROM inbound_documents
|
||||
WHERE user_id = ${userId}
|
||||
`;
|
||||
const usedBytes = Number(rows[0]?.used ?? 0);
|
||||
if (usedBytes + bytes.byteLength > limits.quotaBytes) {
|
||||
throw new QuotaExceededError(usedBytes);
|
||||
}
|
||||
|
||||
const { content, codec } = encryptDocumentContent(bytes);
|
||||
|
||||
// No mass assignment — every column is set field-by-field; `userId`
|
||||
// comes from the session, never the body. `documentDate` defaults to
|
||||
// the upload day so display == sort == filter (user-editable later).
|
||||
const created = await tx.inboundDocument.create({
|
||||
data: {
|
||||
userId,
|
||||
kind: input.kind ?? "OTHER",
|
||||
// Stored plaintext on purpose (mirrors `filename`) so the list can
|
||||
// ILIKE-search + ORDER BY it. It MAY hold PHI the user types; that
|
||||
// is the accepted tradeoff for server-side search/sort — the
|
||||
// document body stays encrypted.
|
||||
title: input.title,
|
||||
filename: input.filename ? input.filename.slice(0, 255) : null,
|
||||
mimeType: detected.mimeType,
|
||||
byteSize: bytes.byteLength,
|
||||
contentEncrypted: content,
|
||||
contentCodec: codec,
|
||||
contentSha256,
|
||||
status: "STORED",
|
||||
documentDate: input.documentDate
|
||||
? isoDateToUtc(input.documentDate)
|
||||
: new Date(),
|
||||
sourceSystem,
|
||||
sourceId,
|
||||
sourceInstance,
|
||||
aiReadDeferred: input.aiDeferred,
|
||||
},
|
||||
omit: { contentEncrypted: true },
|
||||
});
|
||||
|
||||
if (episodeIds.length > 0) {
|
||||
await linkTargets(tx, {
|
||||
userId,
|
||||
sourceKind: "document",
|
||||
sourceId: created.id,
|
||||
targetKind: "conditionEpisode",
|
||||
targetIds: episodeIds,
|
||||
});
|
||||
}
|
||||
if (encounterIds.length > 0) {
|
||||
await linkTargets(tx, {
|
||||
userId,
|
||||
sourceKind: "document",
|
||||
sourceId: created.id,
|
||||
targetKind: "encounter",
|
||||
targetIds: encounterIds,
|
||||
});
|
||||
}
|
||||
return created;
|
||||
});
|
||||
} catch (err) {
|
||||
if (err instanceof QuotaExceededError) {
|
||||
return {
|
||||
kind: "quotaExceeded",
|
||||
usedBytes: err.usedBytes,
|
||||
quotaBytes: limits.quotaBytes,
|
||||
};
|
||||
}
|
||||
if (isP2002(err)) {
|
||||
// A racing upload won one of the two partial unique indexes — the same
|
||||
// source key, or the same bytes. Surface the winner exactly as the fast
|
||||
// paths above would have.
|
||||
if (sourceSystem && sourceId) {
|
||||
const match = await findSourceKey(
|
||||
userId,
|
||||
sourceSystem,
|
||||
sourceId,
|
||||
sourceInstance,
|
||||
);
|
||||
if (match) return { kind: "sourceKey", match };
|
||||
}
|
||||
const winner = await prisma.inboundDocument.findFirst({
|
||||
where: { userId, contentSha256, deletedAt: null },
|
||||
omit: { contentEncrypted: true },
|
||||
});
|
||||
if (winner) return answerDuplicate(input, winner);
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
|
||||
await auditLog("documents.inbound.store", {
|
||||
userId,
|
||||
ipAddress: input.ipAddress,
|
||||
details: {
|
||||
documentId: document.id,
|
||||
mime: detected.mimeType,
|
||||
...(input.scoped ? { scoped: true } : {}),
|
||||
...(sourceSystem ? { sourceSystem } : {}),
|
||||
},
|
||||
});
|
||||
|
||||
annotate({
|
||||
action: { name: "documents.vault.upload" },
|
||||
meta: {
|
||||
documentId: document.id,
|
||||
byteSize: document.byteSize,
|
||||
servingClass: detected.servingClass,
|
||||
linked: episodeIds.length,
|
||||
linkedVisits: encounterIds.length,
|
||||
scoped: input.scoped,
|
||||
sourceSystem,
|
||||
aiDeferred: input.aiDeferred,
|
||||
},
|
||||
});
|
||||
|
||||
// Auto-index the freshly stored document for content search: enqueue a
|
||||
// fire-and-forget background job (provider-first, local text-layer fallback).
|
||||
// The caller never blocks on or fails because of indexing — the enqueue is
|
||||
// not awaited and swallows its own errors (a missing boss or a transient
|
||||
// send failure is a silent no-op). Only fresh inserts enqueue — a duplicate
|
||||
// returns early above and never reaches here.
|
||||
//
|
||||
// `aiRead=defer` narrows it to the local text layer: an import of a whole
|
||||
// archive must not turn into one provider call per file. The person reads
|
||||
// them later, deliberately, from the document itself.
|
||||
if (input.aiDeferred) {
|
||||
void enqueueDocumentIndex(userId, document.id, { localOnly: true });
|
||||
} else {
|
||||
void enqueueDocumentIndex(userId, document.id);
|
||||
}
|
||||
|
||||
// Render a preview thumbnail in the background too (pure local compute — no
|
||||
// egress). Same fire-and-forget contract; a dropped enqueue is recoverable
|
||||
// via the boot backfill.
|
||||
void enqueueDocumentThumbnail(userId, document.id);
|
||||
|
||||
// Summarise the freshly stored document in the background — but ONLY when the
|
||||
// `documentsAutoAiRead` opt-in is ON (the job re-checks it and the egress
|
||||
// consent, and no-ops otherwise) and the `documentAi` capability is open for
|
||||
// this record. Storing never depends on AI: the document is stored above
|
||||
// whatever the capability says, and the index job still runs its
|
||||
// provider-free text-layer path.
|
||||
//
|
||||
// A deferred upload skips it outright and keeps `summaryState: NONE`, so
|
||||
// the detail sheet offers "Generate summary" rather than a pending state.
|
||||
const documentAi = input.aiDeferred
|
||||
? { available: false as const, reason: "deferred" }
|
||||
: await input.documentAi();
|
||||
if (documentAi.available) {
|
||||
void enqueueDocumentSummary(userId, document.id);
|
||||
} else {
|
||||
annotate({
|
||||
action: { name: "documents.summary.enqueueSkipped" },
|
||||
meta: { documentId: document.id, reason: documentAi.reason },
|
||||
});
|
||||
}
|
||||
|
||||
return {
|
||||
kind: "stored",
|
||||
document,
|
||||
servingClass: detected.servingClass,
|
||||
linkedEpisodes: episodeIds.length,
|
||||
linkedVisits: encounterIds.length,
|
||||
};
|
||||
}
|
||||
+130
-12
@@ -11,8 +11,21 @@
|
||||
* 3. `DocumentImportKey` — a key whose document the purge has removed.
|
||||
*
|
||||
* The first two resolve to a document (live → duplicate, tombstoned →
|
||||
* deleted); the third only to "deleted". The upload route and the lookup
|
||||
* route both ask here, so they cannot disagree about what a key means.
|
||||
* deleted); the third only to "deleted". The upload route, the lookup route
|
||||
* and the document picker all ask here, so they cannot disagree about what a
|
||||
* key means.
|
||||
*
|
||||
* v1.39.3 — a key also names its instance (`sourceInstance`, the origin of
|
||||
* the Paperless-ngx or Papra it came from): id 123 on one Paperless and id
|
||||
* 123 on another are different documents. A null on either side is a
|
||||
* wildcard, so nothing stored before the column existed is forgotten:
|
||||
*
|
||||
* - a key stored without an instance (a v1.39.2 import, an older script, a
|
||||
* workflow that does not send one) matches a lookup for any instance of
|
||||
* its system;
|
||||
* - a lookup without an instance matches a key stored for any instance.
|
||||
*
|
||||
* Where both an exact and a wildcard match exist, the exact one wins.
|
||||
*/
|
||||
import { prisma } from "@/lib/db";
|
||||
import { checkRateLimit, type RateLimitResult } from "@/lib/rate-limit";
|
||||
@@ -23,22 +36,44 @@ export type SourceKeyMatch =
|
||||
| { state: "live"; document: SerialisableDocument }
|
||||
| { state: "deleted"; id: string | null };
|
||||
|
||||
/**
|
||||
* The instance half of a key match: the same instance or a stored key without
|
||||
* one; any instance when the lookup has none.
|
||||
*/
|
||||
export function instanceMatch(sourceInstance: string | null): {
|
||||
OR?: Array<{ sourceInstance: string | null }>;
|
||||
} {
|
||||
return sourceInstance === null
|
||||
? {}
|
||||
: { OR: [{ sourceInstance }, { sourceInstance: null }] };
|
||||
}
|
||||
|
||||
/** Exact instance first, then a key stored without one (nulls sort last). */
|
||||
const EXACT_FIRST = { sourceInstance: { sort: "asc", nulls: "last" } } as const;
|
||||
|
||||
export async function findSourceKey(
|
||||
userId: string,
|
||||
sourceSystem: DocumentSourceSystemValue,
|
||||
sourceId: string,
|
||||
sourceInstance: string | null = null,
|
||||
): Promise<SourceKeyMatch | null> {
|
||||
const key = {
|
||||
userId,
|
||||
sourceSystem,
|
||||
sourceId,
|
||||
...instanceMatch(sourceInstance),
|
||||
};
|
||||
const own = await prisma.inboundDocument.findFirst({
|
||||
where: { userId, sourceSystem, sourceId },
|
||||
where: key,
|
||||
omit: { contentEncrypted: true },
|
||||
orderBy: EXACT_FIRST,
|
||||
});
|
||||
const viaAlias = own
|
||||
? null
|
||||
: await prisma.documentSourceAlias.findUnique({
|
||||
where: {
|
||||
userId_sourceSystem_sourceId: { userId, sourceSystem, sourceId },
|
||||
},
|
||||
: await prisma.documentSourceAlias.findFirst({
|
||||
where: key,
|
||||
select: { document: { omit: { contentEncrypted: true } } },
|
||||
orderBy: EXACT_FIRST,
|
||||
});
|
||||
const document = own ?? viaAlias?.document ?? null;
|
||||
if (document) {
|
||||
@@ -46,10 +81,8 @@ export async function findSourceKey(
|
||||
? { state: "deleted", id: document.id }
|
||||
: { state: "live", document };
|
||||
}
|
||||
const purged = await prisma.documentImportKey.findUnique({
|
||||
where: {
|
||||
userId_sourceSystem_sourceId: { userId, sourceSystem, sourceId },
|
||||
},
|
||||
const purged = await prisma.documentImportKey.findFirst({
|
||||
where: key,
|
||||
select: { id: true },
|
||||
});
|
||||
return purged ? { state: "deleted", id: null } : null;
|
||||
@@ -74,13 +107,14 @@ export async function rememberSourceAlias(
|
||||
documentId: string,
|
||||
sourceSystem: DocumentSourceSystemValue,
|
||||
sourceId: string,
|
||||
sourceInstance: string | null = null,
|
||||
): Promise<"remembered" | "known" | "limit"> {
|
||||
const held = await prisma.documentSourceAlias.count({
|
||||
where: { userId, documentId },
|
||||
});
|
||||
if (held >= MAX_SOURCE_ALIASES_PER_DOCUMENT) return "limit";
|
||||
const { count } = await prisma.documentSourceAlias.createMany({
|
||||
data: [{ userId, documentId, sourceSystem, sourceId }],
|
||||
data: [{ userId, documentId, sourceSystem, sourceInstance, sourceId }],
|
||||
skipDuplicates: true,
|
||||
});
|
||||
return count > 0 ? "remembered" : "known";
|
||||
@@ -107,3 +141,87 @@ export function checkSourceLookupRateLimit(
|
||||
SOURCE_LOOKUP_WINDOW_MS,
|
||||
);
|
||||
}
|
||||
|
||||
/** What the vault holds for one source key, as the picker lists it. */
|
||||
export type SourceKeyState =
|
||||
{ state: "imported"; documentId: string } | { state: "deleted" };
|
||||
|
||||
/**
|
||||
* {@link findSourceKey} for a page of results at once: the same three places,
|
||||
* three queries for the whole page instead of three per result. A key held by
|
||||
* a live document answers `imported` with its id; one held by a tombstone, or
|
||||
* kept by the purge ledger, answers `deleted`. A key the vault does not know
|
||||
* is absent from the map.
|
||||
*/
|
||||
export async function findSourceKeyStates(
|
||||
userId: string,
|
||||
sourceSystem: DocumentSourceSystemValue,
|
||||
sourceIds: string[],
|
||||
sourceInstance: string | null = null,
|
||||
): Promise<Map<string, SourceKeyState>> {
|
||||
const states = new Map<string, SourceKeyState>();
|
||||
const ids = [...new Set(sourceIds)];
|
||||
if (ids.length === 0) return states;
|
||||
|
||||
const where = {
|
||||
userId,
|
||||
sourceSystem,
|
||||
sourceId: { in: ids },
|
||||
...instanceMatch(sourceInstance),
|
||||
};
|
||||
const [own, aliases, purged] = await Promise.all([
|
||||
prisma.inboundDocument.findMany({
|
||||
where,
|
||||
select: {
|
||||
id: true,
|
||||
sourceId: true,
|
||||
sourceInstance: true,
|
||||
deletedAt: true,
|
||||
},
|
||||
}),
|
||||
prisma.documentSourceAlias.findMany({
|
||||
where,
|
||||
select: {
|
||||
sourceId: true,
|
||||
sourceInstance: true,
|
||||
document: { select: { id: true, deletedAt: true } },
|
||||
},
|
||||
}),
|
||||
prisma.documentImportKey.findMany({
|
||||
where,
|
||||
select: { sourceId: true },
|
||||
}),
|
||||
]);
|
||||
// A wildcard (instance-less) row first, so an exact-instance row after it
|
||||
// wins, as it does in `findSourceKey`.
|
||||
const wildcardFirst = <T extends { sourceInstance: string | null }>(
|
||||
rows: T[],
|
||||
) =>
|
||||
[...rows].sort(
|
||||
(a, b) =>
|
||||
Number(a.sourceInstance === sourceInstance) -
|
||||
Number(b.sourceInstance === sourceInstance),
|
||||
);
|
||||
|
||||
// Lowest precedence first, so the document's own key wins, as it does in
|
||||
// `findSourceKey`.
|
||||
for (const row of purged) states.set(row.sourceId, { state: "deleted" });
|
||||
for (const row of wildcardFirst(aliases)) {
|
||||
states.set(
|
||||
row.sourceId,
|
||||
row.document.deletedAt
|
||||
? { state: "deleted" }
|
||||
: { state: "imported", documentId: row.document.id },
|
||||
);
|
||||
}
|
||||
for (const row of wildcardFirst(own)) {
|
||||
if (!row.sourceId) continue;
|
||||
states.set(
|
||||
row.sourceId,
|
||||
row.deletedAt
|
||||
? { state: "deleted" }
|
||||
: { state: "imported", documentId: row.id },
|
||||
);
|
||||
}
|
||||
return states;
|
||||
}
|
||||
@@ -0,0 +1,642 @@
|
||||
/**
|
||||
* The Paperless-ngx and Papra adapters against recorded response shapes, with
|
||||
* the network mocked at `safeFetch`. What these pin:
|
||||
*
|
||||
* - every request carries the listed origin as the operator-approved pin and
|
||||
* forbids redirects, and a 3xx is reported, not followed;
|
||||
* - the source's own query parameters (name, tag, date, page), and the local
|
||||
* re-check that keeps a server ignoring a filter from widening a result;
|
||||
* - the error mapping (401 → authRefused, 406 / an old X-Api-Version →
|
||||
* versionTooOld, 404 → notFound, non-JSON → badResponse, a refused or dead
|
||||
* host → unreachable);
|
||||
* - the byte caps on JSON and on a download.
|
||||
*/
|
||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
|
||||
const safeFetch = vi.hoisted(() => vi.fn());
|
||||
|
||||
vi.mock("@/lib/safe-fetch", async (importOriginal) => {
|
||||
const actual = await importOriginal<typeof import("@/lib/safe-fetch")>();
|
||||
return { ...actual, safeFetch };
|
||||
});
|
||||
|
||||
import { SafeFetchError } from "@/lib/safe-fetch";
|
||||
|
||||
import { sourceClient } from "../clients";
|
||||
import {
|
||||
DocumentSourceError,
|
||||
DocumentSourceTooLargeError,
|
||||
SOURCE_JSON_MAX_BYTES,
|
||||
} from "../http";
|
||||
|
||||
const PAPERLESS = {
|
||||
system: "PAPERLESS" as const,
|
||||
origin: "http://paperless.lan:8000",
|
||||
baseUrl: "http://paperless.lan:8000/paperless",
|
||||
organizationId: null,
|
||||
token: "pl-secret-token",
|
||||
};
|
||||
|
||||
const PAPRA = {
|
||||
system: "PAPRA" as const,
|
||||
origin: "https://papra.example.com",
|
||||
baseUrl: "https://papra.example.com",
|
||||
organizationId: "org_a1b2c3d4e5f6g7h8i9j0k1l2",
|
||||
token: "ppk_secret",
|
||||
};
|
||||
|
||||
function json(body: unknown, init: ResponseInit = {}): Response {
|
||||
return new Response(JSON.stringify(body), {
|
||||
status: 200,
|
||||
headers: { "content-type": "application/json" },
|
||||
...init,
|
||||
});
|
||||
}
|
||||
|
||||
/** Route mocked requests by path; anything unmatched is a 404. */
|
||||
function serve(routes: Record<string, (url: URL) => Response>) {
|
||||
safeFetch.mockImplementation(async (target: string) => {
|
||||
const url = new URL(target);
|
||||
for (const [prefix, handler] of Object.entries(routes)) {
|
||||
if (url.pathname.startsWith(prefix)) return handler(url);
|
||||
}
|
||||
return new Response("not found", { status: 404 });
|
||||
});
|
||||
}
|
||||
|
||||
function calledUrls(): URL[] {
|
||||
return safeFetch.mock.calls.map((c) => new URL(String(c[0])));
|
||||
}
|
||||
|
||||
async function failure(promise: Promise<unknown>) {
|
||||
try {
|
||||
await promise;
|
||||
} catch (err) {
|
||||
return err;
|
||||
}
|
||||
throw new Error("expected a failure");
|
||||
}
|
||||
|
||||
beforeEach(() => {
|
||||
safeFetch.mockReset();
|
||||
});
|
||||
|
||||
describe("every request is pinned to the listed origin", () => {
|
||||
it("passes the origin as the operator-approved pin and forbids redirects", async () => {
|
||||
serve({ "/paperless/api/documents/": () => json({ results: [] }) });
|
||||
await sourceClient(PAPERLESS).test();
|
||||
expect(safeFetch).toHaveBeenCalledTimes(1);
|
||||
const [target, init, opts] = safeFetch.mock.calls[0];
|
||||
expect(String(target)).toBe(
|
||||
"http://paperless.lan:8000/paperless/api/documents/?page_size=1&fields=id",
|
||||
);
|
||||
expect(init.redirect).toBe("manual");
|
||||
expect(init.headers.Authorization).toBe("Token pl-secret-token");
|
||||
expect(init.headers.Accept).toBe("application/json; version=9");
|
||||
expect(opts.operatorApprovedPrivateOrigin).toBe(
|
||||
"http://paperless.lan:8000",
|
||||
);
|
||||
expect(opts.timeoutMs).toBeGreaterThan(0);
|
||||
expect(opts.requirePublicHost).toBeUndefined();
|
||||
});
|
||||
|
||||
it("reports a redirect instead of following it", async () => {
|
||||
serve({
|
||||
"/paperless/api/documents/": () =>
|
||||
new Response(null, {
|
||||
status: 302,
|
||||
headers: { location: "http://169.254.169.254/latest/meta-data" },
|
||||
}),
|
||||
});
|
||||
const err = await failure(sourceClient(PAPERLESS).test());
|
||||
expect(err).toBeInstanceOf(DocumentSourceError);
|
||||
expect((err as DocumentSourceError).code).toBe("redirected");
|
||||
expect(safeFetch).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("reports a refused or dead destination as unreachable", async () => {
|
||||
safeFetch.mockRejectedValue(new SafeFetchError("refused", "private_host"));
|
||||
const err = await failure(sourceClient(PAPRA).test());
|
||||
expect((err as DocumentSourceError).code).toBe("unreachable");
|
||||
});
|
||||
});
|
||||
|
||||
describe("Paperless-ngx", () => {
|
||||
const tags = {
|
||||
results: [
|
||||
{ id: 7, name: "Health" },
|
||||
{ id: 9, name: "Insurance" },
|
||||
],
|
||||
next: null,
|
||||
};
|
||||
|
||||
it("searches by title with tag, date and page, and names the tags", async () => {
|
||||
serve({
|
||||
"/paperless/api/tags/": () => json(tags),
|
||||
"/paperless/api/documents/": () =>
|
||||
json({
|
||||
count: 30,
|
||||
next: "http://paperless.lan:8000/paperless/api/documents/?page=3",
|
||||
results: [
|
||||
{
|
||||
id: 41,
|
||||
title: "Blood test",
|
||||
created: "2025-03-04",
|
||||
tags: [7],
|
||||
original_file_name: "labor.pdf",
|
||||
},
|
||||
// A server that ignored the tag filter must not widen the result.
|
||||
{ id: 42, title: "Car", created: "2025-03-05", tags: [9] },
|
||||
// Nor one that ignored the date filter.
|
||||
{ id: 43, title: "Old", created: "2019-01-01", tags: [7] },
|
||||
// No title: the file name without its extension.
|
||||
{
|
||||
id: 44,
|
||||
title: " ",
|
||||
created: "2025-06-01T10:00:00+02:00",
|
||||
tags: [7],
|
||||
original_file_name: "discharge-letter.pdf",
|
||||
},
|
||||
],
|
||||
}),
|
||||
});
|
||||
const result = await sourceClient(PAPERLESS).search({
|
||||
q: "blood",
|
||||
tagId: "7",
|
||||
from: "2025-01-01",
|
||||
to: "2025-12-31",
|
||||
page: 2,
|
||||
});
|
||||
|
||||
const list = calledUrls().find((u) => u.pathname.endsWith("/documents/"));
|
||||
expect(list?.searchParams.get("title__icontains")).toBe("blood");
|
||||
expect(list?.searchParams.get("tags__id__all")).toBe("7");
|
||||
expect(list?.searchParams.get("created__gte")).toBe("2025-01-01");
|
||||
expect(list?.searchParams.get("created__lte")).toBe("2025-12-31");
|
||||
// API version 9 made `created` a date; the `__date` lookups are gone.
|
||||
expect(list?.searchParams.has("created__date__gte")).toBe(false);
|
||||
expect(list?.searchParams.get("page")).toBe("2");
|
||||
expect(list?.searchParams.get("page_size")).toBe("25");
|
||||
|
||||
expect(result.hasMore).toBe(true);
|
||||
expect(result.items).toEqual([
|
||||
{
|
||||
sourceId: "41",
|
||||
title: "Blood test",
|
||||
date: "2025-03-04",
|
||||
tags: ["Health"],
|
||||
sizeBytes: null,
|
||||
},
|
||||
{
|
||||
sourceId: "44",
|
||||
title: "discharge-letter",
|
||||
date: "2025-06-01",
|
||||
tags: ["Health"],
|
||||
sizeBytes: null,
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
||||
it("answers a page past the end with nothing", async () => {
|
||||
serve({
|
||||
"/paperless/api/tags/": () => json(tags),
|
||||
"/paperless/api/documents/": () =>
|
||||
json({ detail: "Invalid page." }, { status: 404 }),
|
||||
});
|
||||
await expect(
|
||||
sourceClient(PAPERLESS).search({
|
||||
q: "",
|
||||
tagId: null,
|
||||
from: null,
|
||||
to: null,
|
||||
page: 99,
|
||||
}),
|
||||
).resolves.toEqual({ items: [], hasMore: false });
|
||||
});
|
||||
|
||||
it("maps a refused token, an unsupported version and an old server", async () => {
|
||||
serve({ "/paperless/api/": () => json({}, { status: 401 }) });
|
||||
expect(
|
||||
((await failure(sourceClient(PAPERLESS).test())) as DocumentSourceError)
|
||||
.code,
|
||||
).toBe("authRefused");
|
||||
|
||||
serve({ "/paperless/api/": () => json({}, { status: 406 }) });
|
||||
expect(
|
||||
((await failure(sourceClient(PAPERLESS).test())) as DocumentSourceError)
|
||||
.code,
|
||||
).toBe("versionTooOld");
|
||||
|
||||
serve({
|
||||
"/paperless/api/": () =>
|
||||
json({ results: [] }, { headers: { "x-api-version": "5" } }),
|
||||
});
|
||||
expect(
|
||||
((await failure(sourceClient(PAPERLESS).test())) as DocumentSourceError)
|
||||
.code,
|
||||
).toBe("versionTooOld");
|
||||
});
|
||||
|
||||
it("reports a web page where the API should be as a bad response", async () => {
|
||||
serve({
|
||||
"/paperless/api/": () =>
|
||||
new Response("<html>login</html>", {
|
||||
headers: { "content-type": "text/html" },
|
||||
}),
|
||||
});
|
||||
const err = await failure(sourceClient(PAPERLESS).test());
|
||||
expect((err as DocumentSourceError).code).toBe("badResponse");
|
||||
});
|
||||
|
||||
it("refuses a JSON answer larger than the cap", async () => {
|
||||
serve({
|
||||
"/paperless/api/": () =>
|
||||
new Response("x".repeat(SOURCE_JSON_MAX_BYTES + 10), { status: 200 }),
|
||||
});
|
||||
const err = await failure(sourceClient(PAPERLESS).test());
|
||||
expect((err as DocumentSourceError).code).toBe("badResponse");
|
||||
});
|
||||
|
||||
it("reads a document's details and downloads the original, within the cap", async () => {
|
||||
serve({
|
||||
"/paperless/api/documents/41/download/": () =>
|
||||
new Response(new Uint8Array([37, 80, 68, 70]), {
|
||||
headers: { "content-disposition": 'attachment; filename="x.pdf"' },
|
||||
}),
|
||||
"/paperless/api/documents/41/": () =>
|
||||
json({
|
||||
id: 41,
|
||||
title: "Blood test",
|
||||
created: "2025-03-04",
|
||||
original_file_name: "labor.pdf",
|
||||
}),
|
||||
});
|
||||
const client = sourceClient(PAPERLESS);
|
||||
await expect(client.document("41")).resolves.toEqual({
|
||||
title: "Blood test",
|
||||
date: "2025-03-04",
|
||||
filename: "labor.pdf",
|
||||
});
|
||||
const file = await client.download("41", 1024);
|
||||
expect([...file.bytes]).toEqual([37, 80, 68, 70]);
|
||||
expect(file.filename).toBe("x.pdf");
|
||||
const download = calledUrls().at(-1);
|
||||
expect(download?.searchParams.get("original")).toBe("true");
|
||||
});
|
||||
|
||||
it("refuses a download larger than the vault accepts, declared or not", async () => {
|
||||
serve({
|
||||
"/paperless/api/documents/41/download/": () =>
|
||||
new Response(new Uint8Array(2048), {
|
||||
headers: { "content-length": "2048" },
|
||||
}),
|
||||
});
|
||||
expect(
|
||||
await failure(sourceClient(PAPERLESS).download("41", 1024)),
|
||||
).toBeInstanceOf(DocumentSourceTooLargeError);
|
||||
|
||||
serve({
|
||||
"/paperless/api/documents/41/download/": () =>
|
||||
new Response(
|
||||
new ReadableStream({
|
||||
start(controller) {
|
||||
controller.enqueue(new Uint8Array(800));
|
||||
controller.enqueue(new Uint8Array(800));
|
||||
controller.close();
|
||||
},
|
||||
}),
|
||||
),
|
||||
});
|
||||
expect(
|
||||
await failure(sourceClient(PAPERLESS).download("41", 1024)),
|
||||
).toBeInstanceOf(DocumentSourceTooLargeError);
|
||||
});
|
||||
});
|
||||
|
||||
describe("ids never shape a request path", () => {
|
||||
it("refuses a Paperless id that is not a number, and dot segments, before dialling", async () => {
|
||||
const client = sourceClient(PAPERLESS);
|
||||
for (const id of ["..", ".", "41/../x", "abc", "41?x=1"]) {
|
||||
const err = await failure(client.document(id));
|
||||
expect((err as DocumentSourceError).code, id).toBe("notFound");
|
||||
expect(
|
||||
((await failure(client.download(id, 10))) as DocumentSourceError).code,
|
||||
).toBe("notFound");
|
||||
}
|
||||
expect(safeFetch).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("refuses a Papra id or organization id outside [A-Za-z0-9_-]", async () => {
|
||||
for (const id of ["..", "doc/1", "doc%2F1", "doc 1"]) {
|
||||
await failure(sourceClient(PAPRA).document(id));
|
||||
}
|
||||
expect(() =>
|
||||
sourceClient({ ...PAPRA, organizationId: "../admin" }),
|
||||
).toThrow(DocumentSourceError);
|
||||
expect(safeFetch).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("leaves a result with an unusable id out of the list", async () => {
|
||||
serve({
|
||||
"/paperless/api/tags/": () => json({ results: [], next: null }),
|
||||
"/paperless/api/documents/": () =>
|
||||
json({
|
||||
next: null,
|
||||
results: [
|
||||
{ id: "..", title: "Odd", created: "2025-01-01", tags: [] },
|
||||
{ id: 5, title: "Fine", created: "2025-01-01", tags: [] },
|
||||
],
|
||||
}),
|
||||
});
|
||||
const result = await sourceClient(PAPERLESS).search({
|
||||
q: "",
|
||||
tagId: null,
|
||||
from: null,
|
||||
to: null,
|
||||
page: 1,
|
||||
});
|
||||
expect(result.items.map((i) => i.sourceId)).toEqual(["5"]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("Paperless-ngx tags", () => {
|
||||
it("pages through tags beyond the first 250", async () => {
|
||||
serve({
|
||||
"/paperless/api/tags/": (url) => {
|
||||
const page = Number(url.searchParams.get("page"));
|
||||
const results = Array.from(
|
||||
{ length: page === 1 ? 250 : 3 },
|
||||
(_, i) => ({
|
||||
id: page * 1000 + i,
|
||||
name: `t${page}-${i}`,
|
||||
}),
|
||||
);
|
||||
return json({ results, next: page === 1 ? "more" : null });
|
||||
},
|
||||
});
|
||||
const tags = await sourceClient(PAPERLESS).tags();
|
||||
expect(tags).toHaveLength(253);
|
||||
expect(calledUrls().map((u) => u.searchParams.get("page"))).toEqual([
|
||||
"1",
|
||||
"2",
|
||||
]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("Papra, as it answers for real", () => {
|
||||
it("quotes each word so a leading dash is not a negation, and sends the date range", async () => {
|
||||
serve({
|
||||
"/api/organizations/org_a1b2c3d4e5f6g7h8i9j0k1l2/documents": () =>
|
||||
json({ documents: [], documentsCount: 0 }),
|
||||
});
|
||||
await sourceClient(PAPRA).search({
|
||||
q: "-Knie Entlass",
|
||||
tagId: null,
|
||||
from: "2024-01-01",
|
||||
to: "2024-12-31",
|
||||
page: 1,
|
||||
});
|
||||
const list = calledUrls().find((u) => u.pathname.endsWith("/documents"));
|
||||
expect(list?.searchParams.get("searchQuery")).toBe(
|
||||
'"-Knie" "Entlass" date:>=2024-01-01 date:<=2024-12-31',
|
||||
);
|
||||
});
|
||||
|
||||
it("dates a search row by its document date only, as Papra's date filter does", async () => {
|
||||
serve({
|
||||
"/api/organizations/org_a1b2c3d4e5f6g7h8i9j0k1l2/documents": () =>
|
||||
json({
|
||||
documentsCount: 2,
|
||||
documents: [
|
||||
{
|
||||
id: "doc_dated",
|
||||
name: "A.pdf",
|
||||
documentDate: "2024-05-01",
|
||||
createdAt: "2025-01-01T00:00:00.000Z",
|
||||
},
|
||||
{
|
||||
id: "doc_undated",
|
||||
name: "B.pdf",
|
||||
createdAt: "2025-01-01T00:00:00.000Z",
|
||||
},
|
||||
],
|
||||
}),
|
||||
});
|
||||
const all = await sourceClient(PAPRA).search({
|
||||
q: "",
|
||||
tagId: null,
|
||||
from: null,
|
||||
to: null,
|
||||
page: 1,
|
||||
});
|
||||
expect(all.items.map((i) => [i.sourceId, i.date])).toEqual([
|
||||
["doc_dated", "2024-05-01"],
|
||||
["doc_undated", null],
|
||||
]);
|
||||
const ranged = await sourceClient(PAPRA).search({
|
||||
q: "",
|
||||
tagId: null,
|
||||
from: "2024-01-01",
|
||||
to: "2025-12-31",
|
||||
page: 1,
|
||||
});
|
||||
// Papra's date: filter never matches a document without a document date;
|
||||
// the per-row check agrees instead of re-admitting it by upload day.
|
||||
expect(ranged.items.map((i) => i.sourceId)).toEqual(["doc_dated"]);
|
||||
});
|
||||
|
||||
it("still files an undated Papra document under the day it was added on import", async () => {
|
||||
serve({
|
||||
"/api/organizations/org_a1b2c3d4e5f6g7h8i9j0k1l2/documents/doc_undated":
|
||||
() =>
|
||||
json({
|
||||
document: {
|
||||
id: "doc_undated",
|
||||
name: "B.pdf",
|
||||
createdAt: "2025-01-01T09:00:00.000Z",
|
||||
},
|
||||
}),
|
||||
});
|
||||
await expect(
|
||||
sourceClient(PAPRA).document("doc_undated"),
|
||||
).resolves.toMatchObject({
|
||||
date: "2025-01-01",
|
||||
});
|
||||
});
|
||||
|
||||
it("refuses an organization id Papra would not issue", () => {
|
||||
for (const org of ["org_123", "../admin", "ORG_A1B2C3D4E5F6G7H8I9J0K1L2"]) {
|
||||
expect(() => sourceClient({ ...PAPRA, organizationId: org })).toThrow(
|
||||
DocumentSourceError,
|
||||
);
|
||||
}
|
||||
expect(safeFetch).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("reads 'not in this organization' and a malformed id as no such organization", async () => {
|
||||
serve({
|
||||
"/api/organizations/": () =>
|
||||
json({ error: { code: "user.not_in_organization" } }, { status: 403 }),
|
||||
});
|
||||
expect(
|
||||
((await failure(sourceClient(PAPRA).test())) as DocumentSourceError).code,
|
||||
).toBe("notFound");
|
||||
serve({
|
||||
"/api/organizations/": () =>
|
||||
json(
|
||||
{ error: { code: "server.invalid_request.params" } },
|
||||
{ status: 400 },
|
||||
),
|
||||
});
|
||||
expect(
|
||||
((await failure(sourceClient(PAPRA).test())) as DocumentSourceError).code,
|
||||
).toBe("notFound");
|
||||
// Any other refusal of the key is still the key.
|
||||
serve({
|
||||
"/api/organizations/": () =>
|
||||
json({ error: { code: "auth.unauthorized" } }, { status: 401 }),
|
||||
});
|
||||
expect(
|
||||
((await failure(sourceClient(PAPRA).test())) as DocumentSourceError).code,
|
||||
).toBe("authRefused");
|
||||
});
|
||||
|
||||
it("fails the connection test when the key cannot read tags", async () => {
|
||||
serve({
|
||||
"/api/organizations/org_a1b2c3d4e5f6g7h8i9j0k1l2/tags": () =>
|
||||
json({}, { status: 401 }),
|
||||
"/api/organizations/org_a1b2c3d4e5f6g7h8i9j0k1l2/documents": () =>
|
||||
json({ documents: [], documentsCount: 0 }),
|
||||
});
|
||||
const err = await failure(sourceClient(PAPRA).test());
|
||||
expect((err as DocumentSourceError).code).toBe("permissionMissing");
|
||||
});
|
||||
});
|
||||
|
||||
describe("Papra", () => {
|
||||
const tags = {
|
||||
tags: [
|
||||
{ id: "tag_b", name: "Lab results" },
|
||||
{ id: "tag_a", name: "Health" },
|
||||
],
|
||||
};
|
||||
|
||||
it("sends the name and the tag through searchQuery and re-checks both", async () => {
|
||||
serve({
|
||||
"/api/organizations/org_a1b2c3d4e5f6g7h8i9j0k1l2/tags": () => json(tags),
|
||||
"/api/organizations/org_a1b2c3d4e5f6g7h8i9j0k1l2/documents": () =>
|
||||
json({
|
||||
documentsCount: 60,
|
||||
documents: [
|
||||
{
|
||||
id: "doc_1",
|
||||
name: "Blood test.pdf",
|
||||
originalName: "Blood test.pdf",
|
||||
originalSize: 1234,
|
||||
documentDate: "2025-03-04",
|
||||
createdAt: "2025-03-04T09:00:00.000Z",
|
||||
tags: [{ id: "tag_b", name: "Lab results" }],
|
||||
},
|
||||
{
|
||||
id: "doc_2",
|
||||
name: "Other.pdf",
|
||||
createdAt: "2025-03-04T09:00:00.000Z",
|
||||
tags: [{ id: "tag_a", name: "Health" }],
|
||||
},
|
||||
{
|
||||
id: "doc_3",
|
||||
name: "Too old.pdf",
|
||||
documentDate: "2018-01-01",
|
||||
createdAt: "2025-03-04T09:00:00.000Z",
|
||||
tags: [{ id: "tag_b", name: "Lab results" }],
|
||||
},
|
||||
],
|
||||
}),
|
||||
});
|
||||
const result = await sourceClient(PAPRA).search({
|
||||
q: "blood",
|
||||
tagId: "tag_b",
|
||||
from: "2025-01-01",
|
||||
to: null,
|
||||
page: 1,
|
||||
});
|
||||
const list = calledUrls().find((u) => u.pathname.endsWith("/documents"));
|
||||
expect(list?.searchParams.get("searchQuery")).toBe(
|
||||
'"blood" tag:"Lab results" date:>=2025-01-01',
|
||||
);
|
||||
expect(list?.searchParams.get("pageIndex")).toBe("0");
|
||||
expect(list?.searchParams.get("pageSize")).toBe("25");
|
||||
expect(safeFetch.mock.calls[0][1].headers.Authorization).toBe(
|
||||
"Bearer ppk_secret",
|
||||
);
|
||||
expect(result).toEqual({
|
||||
items: [
|
||||
{
|
||||
sourceId: "doc_1",
|
||||
title: "Blood test",
|
||||
date: "2025-03-04",
|
||||
tags: ["Lab results"],
|
||||
sizeBytes: 1234,
|
||||
},
|
||||
],
|
||||
hasMore: true,
|
||||
});
|
||||
});
|
||||
|
||||
it("matches nothing for a tag that no longer exists", async () => {
|
||||
serve({
|
||||
"/api/organizations/org_a1b2c3d4e5f6g7h8i9j0k1l2/tags": () => json(tags),
|
||||
});
|
||||
await expect(
|
||||
sourceClient(PAPRA).search({
|
||||
q: "",
|
||||
tagId: "tag_gone",
|
||||
from: null,
|
||||
to: null,
|
||||
page: 1,
|
||||
}),
|
||||
).resolves.toEqual({ items: [], hasMore: false });
|
||||
expect(calledUrls().some((u) => u.pathname.endsWith("/documents"))).toBe(
|
||||
false,
|
||||
);
|
||||
});
|
||||
|
||||
it("lists tags by name and maps an unknown organization to notFound", async () => {
|
||||
serve({
|
||||
"/api/organizations/org_a1b2c3d4e5f6g7h8i9j0k1l2/tags": () => json(tags),
|
||||
});
|
||||
await expect(sourceClient(PAPRA).tags()).resolves.toEqual([
|
||||
{ id: "tag_a", name: "Health" },
|
||||
{ id: "tag_b", name: "Lab results" },
|
||||
]);
|
||||
|
||||
serve({});
|
||||
const err = await failure(sourceClient(PAPRA).test());
|
||||
expect((err as DocumentSourceError).code).toBe("notFound");
|
||||
});
|
||||
|
||||
it("reads a document and downloads its file", async () => {
|
||||
serve({
|
||||
"/api/organizations/org_a1b2c3d4e5f6g7h8i9j0k1l2/documents/doc_1/file":
|
||||
() => new Response(new Uint8Array([1, 2, 3])),
|
||||
"/api/organizations/org_a1b2c3d4e5f6g7h8i9j0k1l2/documents/doc_1": () =>
|
||||
json({
|
||||
document: {
|
||||
id: "doc_1",
|
||||
name: "Blood test.pdf",
|
||||
originalName: "blood.pdf",
|
||||
documentDate: "2025-02-01",
|
||||
createdAt: "2025-03-04T09:00:00.000Z",
|
||||
},
|
||||
}),
|
||||
});
|
||||
const client = sourceClient(PAPRA);
|
||||
await expect(client.document("doc_1")).resolves.toEqual({
|
||||
title: "Blood test",
|
||||
date: "2025-02-01",
|
||||
filename: "blood.pdf",
|
||||
});
|
||||
const file = await client.download("doc_1", 1024);
|
||||
expect([...file.bytes]).toEqual([1, 2, 3]);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,110 @@
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
|
||||
import {
|
||||
configuredDocumentSourceOrigins,
|
||||
documentSourcesAvailable,
|
||||
evaluateSourceBaseUrl,
|
||||
listedDocumentSourceOrigins,
|
||||
} from "../origins";
|
||||
|
||||
const ORIGINAL = process.env.DOCUMENT_SOURCE_ORIGINS;
|
||||
|
||||
afterEach(() => {
|
||||
if (ORIGINAL === undefined) delete process.env.DOCUMENT_SOURCE_ORIGINS;
|
||||
else process.env.DOCUMENT_SOURCE_ORIGINS = ORIGINAL;
|
||||
vi.restoreAllMocks();
|
||||
});
|
||||
|
||||
describe("DOCUMENT_SOURCE_ORIGINS", () => {
|
||||
it("is off when unset or empty", () => {
|
||||
delete process.env.DOCUMENT_SOURCE_ORIGINS;
|
||||
expect(documentSourcesAvailable()).toBe(false);
|
||||
process.env.DOCUMENT_SOURCE_ORIGINS = " ";
|
||||
expect(documentSourcesAvailable()).toBe(false);
|
||||
});
|
||||
|
||||
it("keeps valid exact origins and drops malformed ones with a redacted warning", () => {
|
||||
const warn = vi.spyOn(console, "warn").mockImplementation(() => {});
|
||||
process.env.DOCUMENT_SOURCE_ORIGINS =
|
||||
"https://paperless.example.com, http://papra.lan:1221, https://user:secret@bad.example/x?token=abc, http://169.254.169.254";
|
||||
const origins = configuredDocumentSourceOrigins();
|
||||
expect([...origins].sort()).toEqual([
|
||||
"http://papra.lan:1221",
|
||||
"https://paperless.example.com",
|
||||
]);
|
||||
expect(documentSourcesAvailable()).toBe(true);
|
||||
const logged = warn.mock.calls.map((c) => String(c[0])).join("\n");
|
||||
expect(logged).toContain("DOCUMENT_SOURCE_ORIGINS");
|
||||
expect(logged).not.toContain("secret");
|
||||
expect(logged).not.toContain("token=abc");
|
||||
});
|
||||
|
||||
it("lists the origins for the settings card", () => {
|
||||
process.env.DOCUMENT_SOURCE_ORIGINS =
|
||||
"https://paperless.example.com,http://papra.lan:1221";
|
||||
expect(listedDocumentSourceOrigins()).toEqual([
|
||||
"http://papra.lan:1221",
|
||||
"https://paperless.example.com",
|
||||
]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("evaluateSourceBaseUrl", () => {
|
||||
const listed = new Set(["https://paperless.example.com", "http://10.0.0.5"]);
|
||||
|
||||
it("accepts a listed origin and keeps a base path without the trailing slash", () => {
|
||||
expect(
|
||||
evaluateSourceBaseUrl("https://paperless.example.com/paperless/", listed),
|
||||
).toEqual({
|
||||
ok: true,
|
||||
origin: "https://paperless.example.com",
|
||||
baseUrl: "https://paperless.example.com/paperless",
|
||||
});
|
||||
expect(evaluateSourceBaseUrl("http://10.0.0.5", listed)).toEqual({
|
||||
ok: true,
|
||||
origin: "http://10.0.0.5",
|
||||
baseUrl: "http://10.0.0.5",
|
||||
});
|
||||
});
|
||||
|
||||
it("refuses an origin that is not listed, public or private", () => {
|
||||
for (const url of [
|
||||
"https://example.org",
|
||||
"http://10.0.0.6",
|
||||
"http://192.168.1.10:8000",
|
||||
"http://169.254.169.254",
|
||||
"http://localhost:8000",
|
||||
]) {
|
||||
expect(evaluateSourceBaseUrl(url, listed), url).toEqual({
|
||||
ok: false,
|
||||
reason: "notAllowed",
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
it("does not turn a listed host into a suffix, sibling port or other scheme grant", () => {
|
||||
for (const url of [
|
||||
"https://evil.paperless.example.com",
|
||||
"https://paperless.example.com:8443",
|
||||
"http://paperless.example.com",
|
||||
"https://paperless.example.com.evil.test",
|
||||
]) {
|
||||
expect(evaluateSourceBaseUrl(url, listed).ok, url).toBe(false);
|
||||
}
|
||||
});
|
||||
|
||||
it("refuses userinfo, a query, a fragment and non-http schemes", () => {
|
||||
for (const url of [
|
||||
"https://user:pass@paperless.example.com",
|
||||
"https://paperless.example.com/?token=x",
|
||||
"https://paperless.example.com/#x",
|
||||
"ftp://paperless.example.com",
|
||||
"not a url",
|
||||
]) {
|
||||
expect(evaluateSourceBaseUrl(url, listed), url).toEqual({
|
||||
ok: false,
|
||||
reason: "invalid",
|
||||
});
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,533 @@
|
||||
/**
|
||||
* The two source adapters the document picker speaks: Paperless-ngx and Papra.
|
||||
*
|
||||
* Each turns the picker's four questions (does this connection work, which
|
||||
* tags exist, which documents match, give me this one) into the source's own
|
||||
* REST calls, and each answer into the picker's minimal shapes. Everything
|
||||
* leaves through `./http.ts`, so the origin pin, the redirect refusal, the
|
||||
* timeouts and the byte caps are not something an adapter can forget.
|
||||
*
|
||||
* Both adapters re-check tag and date filters on what comes back. A server
|
||||
* that ignores a parameter (an older Paperless, a Papra whose search syntax
|
||||
* changed) must not widen a result into documents the person filtered out;
|
||||
* the import script makes the same check for the same reason.
|
||||
*
|
||||
* API versions: Paperless-ngx is asked for API version 9 (Paperless-ngx 2.16
|
||||
* and later, the version where `created` became a plain date; it answers 406
|
||||
* to a version it does not speak). Papra's endpoints are the documented
|
||||
* organization-scoped ones (`/api/organizations/{id}/documents`, `/tags`,
|
||||
* `/documents/{id}`, `/documents/{id}/file`).
|
||||
*/
|
||||
import { DOCUMENT_TITLE_MAX } from "@/lib/validations/inbound-documents";
|
||||
|
||||
import {
|
||||
DocumentSourceError,
|
||||
sourceDownload,
|
||||
sourceJson,
|
||||
type SourceTarget,
|
||||
} from "./http";
|
||||
import {
|
||||
DOCUMENT_PICKER_MAX_TAGS,
|
||||
DOCUMENT_PICKER_PAGE_SIZE,
|
||||
isSourceDocumentId,
|
||||
PAPRA_ORG_ID,
|
||||
type DocumentPickerSystem,
|
||||
type DocumentSourceErrorCode,
|
||||
type DocumentSourceTagDto,
|
||||
} from "./types";
|
||||
|
||||
/** Paperless-ngx API version the picker speaks (Paperless-ngx 2.16+). */
|
||||
export const PAPERLESS_API_VERSION = 9;
|
||||
|
||||
export interface SourceSearchParams {
|
||||
/** Name to search for; empty lists everything. */
|
||||
q: string;
|
||||
/** A tag id as the source names it. */
|
||||
tagId: string | null;
|
||||
/** YYYY-MM-DD, inclusive. */
|
||||
from: string | null;
|
||||
to: string | null;
|
||||
/** 1-based. */
|
||||
page: number;
|
||||
}
|
||||
|
||||
export interface SourceListItem {
|
||||
sourceId: string;
|
||||
title: string;
|
||||
date: string | null;
|
||||
tags: string[];
|
||||
sizeBytes: number | null;
|
||||
}
|
||||
|
||||
export interface SourceDocumentMeta {
|
||||
title: string;
|
||||
date: string | null;
|
||||
filename: string | null;
|
||||
}
|
||||
|
||||
export interface SourceClient {
|
||||
readonly system: DocumentPickerSystem;
|
||||
/**
|
||||
* The instance this client talks to, as the normalised origin every source
|
||||
* key it produces carries (`sourceInstance`).
|
||||
*/
|
||||
readonly instance: string;
|
||||
test(): Promise<void>;
|
||||
tags(): Promise<DocumentSourceTagDto[]>;
|
||||
search(
|
||||
params: SourceSearchParams,
|
||||
): Promise<{ items: SourceListItem[]; hasMore: boolean }>;
|
||||
document(sourceId: string): Promise<SourceDocumentMeta>;
|
||||
download(
|
||||
sourceId: string,
|
||||
maxFileBytes: number,
|
||||
): Promise<{ bytes: Uint8Array; filename: string | null }>;
|
||||
}
|
||||
|
||||
export interface SourceConnection {
|
||||
system: DocumentPickerSystem;
|
||||
origin: string;
|
||||
baseUrl: string;
|
||||
organizationId: string | null;
|
||||
token: string;
|
||||
}
|
||||
|
||||
/** Paperless tags are read this many per page. */
|
||||
const TAG_PAGE_SIZE = 250;
|
||||
|
||||
/** Refuse an id that is not one the system issues, before it reaches a URL. */
|
||||
function requireId(system: DocumentPickerSystem, id: string): void {
|
||||
if (!isSourceDocumentId(system, id)) {
|
||||
throw new DocumentSourceError("notFound");
|
||||
}
|
||||
}
|
||||
|
||||
// ── Defensive readers ──────────────────────────────────────────────────────
|
||||
|
||||
type Json = Record<string, unknown>;
|
||||
|
||||
function asRecord(value: unknown): Json | null {
|
||||
return value && typeof value === "object" && !Array.isArray(value)
|
||||
? (value as Json)
|
||||
: null;
|
||||
}
|
||||
|
||||
function asArray(value: unknown): unknown[] {
|
||||
return Array.isArray(value) ? value : [];
|
||||
}
|
||||
|
||||
function asText(value: unknown): string | null {
|
||||
if (typeof value === "string") return value;
|
||||
if (typeof value === "number" && Number.isFinite(value)) return String(value);
|
||||
return null;
|
||||
}
|
||||
|
||||
/** The body is not the shape the API documents: most often a base address
|
||||
* that points at the web app instead of the instance. */
|
||||
function shapeError(): DocumentSourceError {
|
||||
return new DocumentSourceError("badResponse");
|
||||
}
|
||||
|
||||
/** YYYY-MM-DD from an ISO date or date-time, or null. */
|
||||
export function dayOf(value: unknown): string | null {
|
||||
if (typeof value !== "string") return null;
|
||||
const day = value.slice(0, 10);
|
||||
return /^\d{4}-\d{2}-\d{2}$/.test(day) ? day : null;
|
||||
}
|
||||
|
||||
function stripExtension(name: string | null): string {
|
||||
return name ? name.replace(/\.[A-Za-z0-9]{1,5}$/, "") : "";
|
||||
}
|
||||
|
||||
/**
|
||||
* A title that is not blank and fits the vault: the source's title, else its
|
||||
* file name without the extension, else a placeholder naming the source id.
|
||||
*/
|
||||
export function titleOf(
|
||||
title: string | null,
|
||||
filename: string | null,
|
||||
fallback: string,
|
||||
): string {
|
||||
const picked =
|
||||
(title ?? "").trim() || stripExtension(filename).trim() || fallback;
|
||||
return picked.slice(0, DOCUMENT_TITLE_MAX);
|
||||
}
|
||||
|
||||
/** Is `day` inside the inclusive range? A document without a date is kept
|
||||
* only when no range was asked for. */
|
||||
export function withinRange(
|
||||
day: string | null,
|
||||
from: string | null,
|
||||
to: string | null,
|
||||
): boolean {
|
||||
if (!from && !to) return true;
|
||||
if (!day) return false;
|
||||
if (from && day < from) return false;
|
||||
if (to && day > to) return false;
|
||||
return true;
|
||||
}
|
||||
|
||||
// ── Paperless-ngx ──────────────────────────────────────────────────────────
|
||||
|
||||
function paperless(connection: SourceConnection): SourceClient {
|
||||
const target: SourceTarget = {
|
||||
origin: connection.origin,
|
||||
baseUrl: connection.baseUrl,
|
||||
headers: {
|
||||
Authorization: `Token ${connection.token}`,
|
||||
Accept: `application/json; version=${PAPERLESS_API_VERSION}`,
|
||||
},
|
||||
};
|
||||
const download: SourceTarget = {
|
||||
...target,
|
||||
headers: { Authorization: `Token ${connection.token}` },
|
||||
};
|
||||
|
||||
/** A server that says which API version it speaks, and it is too old. */
|
||||
function checkVersion(response: Response): void {
|
||||
const raw = response.headers.get("x-api-version");
|
||||
if (raw === null) return;
|
||||
const version = Number(raw);
|
||||
if (Number.isFinite(version) && version < PAPERLESS_API_VERSION) {
|
||||
throw new DocumentSourceError("versionTooOld", response.status);
|
||||
}
|
||||
}
|
||||
|
||||
async function tagMap(): Promise<Map<string, string>> {
|
||||
const tags = await listTags();
|
||||
return new Map(tags.map((tag) => [tag.id, tag.name]));
|
||||
}
|
||||
|
||||
async function listTags(): Promise<DocumentSourceTagDto[]> {
|
||||
// Paged by number rather than by following `next`: the link is the
|
||||
// server's to write, the page number is ours.
|
||||
const tags: DocumentSourceTagDto[] = [];
|
||||
for (let pageNo = 1; tags.length < DOCUMENT_PICKER_MAX_TAGS; pageNo += 1) {
|
||||
const query = new URLSearchParams({
|
||||
page: String(pageNo),
|
||||
page_size: String(TAG_PAGE_SIZE),
|
||||
ordering: "name",
|
||||
});
|
||||
const { body, response } = await sourceJson(
|
||||
target,
|
||||
`/api/tags/?${query}`,
|
||||
);
|
||||
checkVersion(response);
|
||||
const page = asRecord(body);
|
||||
if (!page) throw shapeError();
|
||||
for (const raw of asArray(page.results)) {
|
||||
const tag = asRecord(raw);
|
||||
const id = asText(tag?.id);
|
||||
const name = asText(tag?.name);
|
||||
if (id && name) tags.push({ id, name });
|
||||
}
|
||||
if (typeof page.next !== "string" || !page.next) break;
|
||||
}
|
||||
return tags.slice(0, DOCUMENT_PICKER_MAX_TAGS);
|
||||
}
|
||||
|
||||
return {
|
||||
system: "PAPERLESS",
|
||||
instance: connection.origin,
|
||||
|
||||
async test() {
|
||||
const { body, response } = await sourceJson(
|
||||
target,
|
||||
"/api/documents/?page_size=1&fields=id",
|
||||
);
|
||||
checkVersion(response);
|
||||
if (!asRecord(body)) throw shapeError();
|
||||
},
|
||||
|
||||
tags: listTags,
|
||||
|
||||
async search(params) {
|
||||
const query = new URLSearchParams({
|
||||
page: String(params.page),
|
||||
page_size: String(DOCUMENT_PICKER_PAGE_SIZE),
|
||||
ordering: "-created",
|
||||
truncate_content: "true",
|
||||
fields: "id,title,created,added,tags,original_file_name",
|
||||
});
|
||||
if (params.q) query.set("title__icontains", params.q);
|
||||
if (params.tagId) query.set("tags__id__all", params.tagId);
|
||||
// API version 9 made `created` a plain date, so the range filters on
|
||||
// it directly (`created__gte` / `created__lte`), both ends inclusive.
|
||||
if (params.from) query.set("created__gte", params.from);
|
||||
if (params.to) query.set("created__lte", params.to);
|
||||
|
||||
let page: Json | null;
|
||||
let names: Map<string, string>;
|
||||
try {
|
||||
const [list, tags] = await Promise.all([
|
||||
sourceJson(target, `/api/documents/?${query}`),
|
||||
tagMap(),
|
||||
]);
|
||||
checkVersion(list.response);
|
||||
page = asRecord(list.body);
|
||||
names = tags;
|
||||
} catch (err) {
|
||||
// Paperless answers a page past the end with 404 ("Invalid page"):
|
||||
// there is simply nothing more.
|
||||
if (err instanceof DocumentSourceError && err.code === "notFound") {
|
||||
return { items: [], hasMore: false };
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
if (!page) throw shapeError();
|
||||
|
||||
const items: SourceListItem[] = [];
|
||||
for (const raw of asArray(page.results)) {
|
||||
const doc = asRecord(raw);
|
||||
const id = asText(doc?.id);
|
||||
if (!doc || !id || !isSourceDocumentId("PAPERLESS", id)) continue;
|
||||
const tagIds = asArray(doc.tags)
|
||||
.map(asText)
|
||||
.filter((t): t is string => t !== null);
|
||||
if (params.tagId && !tagIds.includes(params.tagId)) continue;
|
||||
const date = dayOf(doc.created) ?? dayOf(doc.added);
|
||||
if (!withinRange(date, params.from, params.to)) continue;
|
||||
items.push({
|
||||
sourceId: id,
|
||||
title: titleOf(
|
||||
asText(doc.title),
|
||||
asText(doc.original_file_name),
|
||||
`Paperless ${id}`,
|
||||
),
|
||||
date,
|
||||
tags: tagIds
|
||||
.map((tagId) => names.get(tagId))
|
||||
.filter((n): n is string => typeof n === "string"),
|
||||
sizeBytes: null,
|
||||
});
|
||||
}
|
||||
return { items, hasMore: typeof page.next === "string" && !!page.next };
|
||||
},
|
||||
|
||||
async document(sourceId) {
|
||||
requireId("PAPERLESS", sourceId);
|
||||
const { body, response } = await sourceJson(
|
||||
target,
|
||||
`/api/documents/${encodeURIComponent(sourceId)}/?fields=id,title,created,added,original_file_name`,
|
||||
);
|
||||
checkVersion(response);
|
||||
const doc = asRecord(body);
|
||||
if (!doc) throw shapeError();
|
||||
const filename = asText(doc.original_file_name);
|
||||
return {
|
||||
title: titleOf(asText(doc.title), filename, `Paperless ${sourceId}`),
|
||||
date: dayOf(doc.created) ?? dayOf(doc.added),
|
||||
filename,
|
||||
};
|
||||
},
|
||||
|
||||
async download(sourceId, maxFileBytes) {
|
||||
requireId("PAPERLESS", sourceId);
|
||||
// The original, not the archived PDF/A: the same bytes a Paperless
|
||||
// workflow or the import script sends, so they dedup against each other.
|
||||
return sourceDownload(
|
||||
download,
|
||||
`/api/documents/${encodeURIComponent(sourceId)}/download/?original=true`,
|
||||
maxFileBytes,
|
||||
);
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
// ── Papra ──────────────────────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Papra's error bodies carry a machine code. Two of them mean "no such
|
||||
* organization" rather than "wrong key": a valid organization id the key's
|
||||
* user is not a member of (403 `user.not_in_organization`), and one Papra
|
||||
* cannot parse (400 `server.invalid_request.params`).
|
||||
*/
|
||||
function papraError(
|
||||
status: number,
|
||||
code: string | null,
|
||||
): DocumentSourceErrorCode | null {
|
||||
if (status === 403 && code === "user.not_in_organization") return "notFound";
|
||||
if (status === 400 && code === "server.invalid_request.params") {
|
||||
return "notFound";
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/** Papra's search syntax quotes a tag name that holds a space. */
|
||||
function papraTagTerm(name: string): string {
|
||||
return /[\s"]/.test(name) ? `tag:${JSON.stringify(name)}` : `tag:${name}`;
|
||||
}
|
||||
|
||||
function papra(connection: SourceConnection): SourceClient {
|
||||
if (
|
||||
!connection.organizationId ||
|
||||
!PAPRA_ORG_ID.test(connection.organizationId)
|
||||
) {
|
||||
throw new DocumentSourceError("notFound");
|
||||
}
|
||||
const org = `/api/organizations/${encodeURIComponent(connection.organizationId)}`;
|
||||
const target: SourceTarget = {
|
||||
origin: connection.origin,
|
||||
baseUrl: connection.baseUrl,
|
||||
headers: {
|
||||
Authorization: `Bearer ${connection.token}`,
|
||||
Accept: "application/json",
|
||||
},
|
||||
classify: papraError,
|
||||
};
|
||||
const download: SourceTarget = {
|
||||
...target,
|
||||
headers: { Authorization: `Bearer ${connection.token}` },
|
||||
};
|
||||
|
||||
async function listTags(): Promise<DocumentSourceTagDto[]> {
|
||||
const { body } = await sourceJson(target, `${org}/tags`);
|
||||
const page = asRecord(body);
|
||||
if (!page) throw shapeError();
|
||||
const tags: DocumentSourceTagDto[] = [];
|
||||
for (const raw of asArray(page.tags)) {
|
||||
const tag = asRecord(raw);
|
||||
const id = asText(tag?.id);
|
||||
const name = asText(tag?.name);
|
||||
if (id && name) tags.push({ id, name });
|
||||
}
|
||||
return tags
|
||||
.sort((a, b) => a.name.localeCompare(b.name))
|
||||
.slice(0, DOCUMENT_PICKER_MAX_TAGS);
|
||||
}
|
||||
|
||||
return {
|
||||
system: "PAPRA",
|
||||
instance: connection.origin,
|
||||
|
||||
async test() {
|
||||
const { body } = await sourceJson(
|
||||
target,
|
||||
`${org}/documents?pageIndex=0&pageSize=1`,
|
||||
);
|
||||
if (!asRecord(body)) throw shapeError();
|
||||
// The tag filter needs `tags:read`; a key without it would save green
|
||||
// and then fail on the first search.
|
||||
try {
|
||||
await sourceJson(target, `${org}/tags`);
|
||||
} catch (err) {
|
||||
if (err instanceof DocumentSourceError && err.code === "authRefused") {
|
||||
throw new DocumentSourceError(
|
||||
"permissionMissing",
|
||||
err.upstreamStatus,
|
||||
);
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
},
|
||||
|
||||
tags: listTags,
|
||||
|
||||
async search(params) {
|
||||
// Each word quoted: Papra reads a leading `-` as a negation and a `:` as
|
||||
// a field, and a quoted word still matches as a prefix.
|
||||
const terms: string[] = params.q
|
||||
.split(/\s+/)
|
||||
.map((word) => word.replace(/"/g, ""))
|
||||
.filter((word) => word.length > 0)
|
||||
.map((word) => `"${word}"`);
|
||||
if (params.tagId) {
|
||||
const tag = (await listTags()).find((t) => t.id === params.tagId);
|
||||
// A tag that no longer exists matches nothing, rather than silently
|
||||
// dropping the filter and listing everything.
|
||||
if (!tag) return { items: [], hasMore: false };
|
||||
terms.push(papraTagTerm(tag.name));
|
||||
}
|
||||
// The range goes to Papra as its own `date:` filter; the check on each
|
||||
// row below stays as the net for a version that reads it differently.
|
||||
if (params.from) terms.push(`date:>=${params.from}`);
|
||||
if (params.to) terms.push(`date:<=${params.to}`);
|
||||
const pageIndex = params.page - 1;
|
||||
const query = new URLSearchParams({
|
||||
pageIndex: String(pageIndex),
|
||||
pageSize: String(DOCUMENT_PICKER_PAGE_SIZE),
|
||||
});
|
||||
if (terms.length > 0) query.set("searchQuery", terms.join(" "));
|
||||
const { body } = await sourceJson(target, `${org}/documents?${query}`);
|
||||
const page = asRecord(body);
|
||||
if (!page) throw shapeError();
|
||||
|
||||
const documents = asArray(page.documents);
|
||||
const items: SourceListItem[] = [];
|
||||
for (const raw of documents) {
|
||||
const doc = asRecord(raw);
|
||||
const id = asText(doc?.id);
|
||||
if (!doc || !id || !isSourceDocumentId("PAPRA", id)) continue;
|
||||
const tags = asArray(doc.tags)
|
||||
.map(asRecord)
|
||||
.filter((t): t is Json => t !== null);
|
||||
if (params.tagId && !tags.some((t) => asText(t.id) === params.tagId)) {
|
||||
continue;
|
||||
}
|
||||
// The row shows the document date only, the field Papra's `date:`
|
||||
// filter reads, so the list and the filter agree: an undated document
|
||||
// shows no date and falls out of any range. The check below is the net
|
||||
// for a server that ignores the filter. The upload day is used only
|
||||
// on import, where a filing date is needed (see `document()`).
|
||||
const date = dayOf(doc.documentDate);
|
||||
if (!withinRange(date, params.from, params.to)) continue;
|
||||
const size = doc.originalSize;
|
||||
items.push({
|
||||
sourceId: id,
|
||||
title: titleOf(
|
||||
stripExtension(asText(doc.name)),
|
||||
asText(doc.originalName),
|
||||
`Papra ${id}`,
|
||||
),
|
||||
date,
|
||||
tags: tags
|
||||
.map((t) => asText(t.name))
|
||||
.filter((n): n is string => n !== null),
|
||||
sizeBytes: typeof size === "number" && size >= 0 ? size : null,
|
||||
});
|
||||
}
|
||||
const total =
|
||||
typeof page.documentsCount === "number" ? page.documentsCount : null;
|
||||
const hasMore =
|
||||
total !== null
|
||||
? (pageIndex + 1) * DOCUMENT_PICKER_PAGE_SIZE < total
|
||||
: documents.length >= DOCUMENT_PICKER_PAGE_SIZE;
|
||||
return { items, hasMore };
|
||||
},
|
||||
|
||||
async document(sourceId) {
|
||||
requireId("PAPRA", sourceId);
|
||||
const { body } = await sourceJson(
|
||||
target,
|
||||
`${org}/documents/${encodeURIComponent(sourceId)}`,
|
||||
);
|
||||
const doc = asRecord(asRecord(body)?.document);
|
||||
if (!doc) throw shapeError();
|
||||
const filename = asText(doc.originalName) ?? asText(doc.name);
|
||||
return {
|
||||
title: titleOf(
|
||||
stripExtension(asText(doc.name)),
|
||||
filename,
|
||||
`Papra ${sourceId}`,
|
||||
),
|
||||
date: dayOf(doc.documentDate) ?? dayOf(doc.createdAt),
|
||||
filename,
|
||||
};
|
||||
},
|
||||
|
||||
async download(sourceId, maxFileBytes) {
|
||||
requireId("PAPRA", sourceId);
|
||||
return sourceDownload(
|
||||
download,
|
||||
`${org}/documents/${encodeURIComponent(sourceId)}/file`,
|
||||
maxFileBytes,
|
||||
);
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export function sourceClient(connection: SourceConnection): SourceClient {
|
||||
return connection.system === "PAPERLESS"
|
||||
? paperless(connection)
|
||||
: papra(connection);
|
||||
}
|
||||
@@ -0,0 +1,117 @@
|
||||
/**
|
||||
* Reading a person's document-source connection, and the one place its token
|
||||
* is decrypted.
|
||||
*
|
||||
* {@link toConnectionDto} is the only shape a connection leaves the server in,
|
||||
* and it has no field that could carry the token: `hasToken` is the literal
|
||||
* `true`. {@link openConnection} decrypts the token for exactly one request to
|
||||
* the source and re-checks the stored origin against the operator's list on
|
||||
* the way, so a connection whose origin was removed from
|
||||
* `DOCUMENT_SOURCE_ORIGINS` answers `originNotAllowed` instead of dialling.
|
||||
*/
|
||||
import { decrypt } from "@/lib/crypto";
|
||||
import { prisma } from "@/lib/db";
|
||||
|
||||
import { sourceClient, type SourceClient } from "./clients";
|
||||
import { DocumentSourceError } from "./http";
|
||||
import { evaluateSourceBaseUrl } from "./origins";
|
||||
import type {
|
||||
DocumentPickerSystem,
|
||||
DocumentSourceConnectionDto,
|
||||
} from "./types";
|
||||
|
||||
/** The columns a DTO or a request needs; never the whole row by accident. */
|
||||
const CONNECTION_SELECT = {
|
||||
system: true,
|
||||
baseUrl: true,
|
||||
organizationId: true,
|
||||
tokenEncrypted: true,
|
||||
lastVerifiedAt: true,
|
||||
} as const;
|
||||
|
||||
export interface StoredConnection {
|
||||
system: string;
|
||||
baseUrl: string;
|
||||
organizationId: string | null;
|
||||
tokenEncrypted: string;
|
||||
lastVerifiedAt: Date | null;
|
||||
}
|
||||
|
||||
export function toConnectionDto(
|
||||
row: Omit<StoredConnection, "tokenEncrypted">,
|
||||
): DocumentSourceConnectionDto {
|
||||
return {
|
||||
system: row.system as DocumentPickerSystem,
|
||||
baseUrl: row.baseUrl,
|
||||
organizationId: row.organizationId,
|
||||
hasToken: true,
|
||||
lastVerifiedAt: row.lastVerifiedAt?.toISOString() ?? null,
|
||||
originAllowed: evaluateSourceBaseUrl(row.baseUrl).ok,
|
||||
};
|
||||
}
|
||||
|
||||
export function listConnections(
|
||||
userId: string,
|
||||
): Promise<Omit<StoredConnection, "tokenEncrypted">[]> {
|
||||
return prisma.documentSourceConnection.findMany({
|
||||
where: { userId },
|
||||
select: {
|
||||
system: true,
|
||||
baseUrl: true,
|
||||
organizationId: true,
|
||||
lastVerifiedAt: true,
|
||||
},
|
||||
orderBy: { system: "asc" },
|
||||
});
|
||||
}
|
||||
|
||||
export function loadConnection(
|
||||
userId: string,
|
||||
system: DocumentPickerSystem,
|
||||
): Promise<StoredConnection | null> {
|
||||
return prisma.documentSourceConnection.findUnique({
|
||||
where: { userId_system: { userId, system } },
|
||||
select: CONNECTION_SELECT,
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* A client for one of the person's stored connections, or the reason there is
|
||||
* none. Throws {@link DocumentSourceError}: `notConnected` when nothing is
|
||||
* stored for the system, `originNotAllowed` when the operator no longer lists
|
||||
* the stored origin.
|
||||
*/
|
||||
export async function openConnection(
|
||||
userId: string,
|
||||
system: DocumentPickerSystem,
|
||||
): Promise<SourceClient> {
|
||||
const row = await loadConnection(userId, system);
|
||||
if (!row) throw new DocumentSourceError("notConnected");
|
||||
return clientFor(row);
|
||||
}
|
||||
|
||||
/** A client for a row already in hand, with the origin re-checked. */
|
||||
export function clientFor(row: StoredConnection): SourceClient {
|
||||
const verdict = evaluateSourceBaseUrl(row.baseUrl);
|
||||
if (!verdict.ok) throw new DocumentSourceError("originNotAllowed");
|
||||
return sourceClient({
|
||||
system: row.system as DocumentPickerSystem,
|
||||
origin: verdict.origin,
|
||||
baseUrl: verdict.baseUrl,
|
||||
organizationId: row.organizationId,
|
||||
token: decrypt(row.tokenEncrypted),
|
||||
});
|
||||
}
|
||||
|
||||
/** Stamp a successful test. */
|
||||
export async function markVerified(
|
||||
userId: string,
|
||||
system: DocumentPickerSystem,
|
||||
): Promise<Date> {
|
||||
const at = new Date();
|
||||
await prisma.documentSourceConnection.updateMany({
|
||||
where: { userId, system },
|
||||
data: { lastVerifiedAt: at },
|
||||
});
|
||||
return at;
|
||||
}
|
||||
@@ -0,0 +1,250 @@
|
||||
/**
|
||||
* The document picker's one way to the network.
|
||||
*
|
||||
* Every request to a Paperless-ngx or Papra instance goes through
|
||||
* {@link sourceRequest}, which goes through `safeFetch` with the connection's
|
||||
* origin as `operatorApprovedPrivateOrigin`:
|
||||
*
|
||||
* - the target must have exactly that origin (a path, query or `next` link
|
||||
* that names another host is refused before a socket opens);
|
||||
* - redirects are never followed; a 3xx is reported as `redirected`, so a
|
||||
* source cannot bounce the request and its `Authorization` header to
|
||||
* another host;
|
||||
* - the connect goes through the pinned operator-approved dispatcher, which
|
||||
* refuses link-local, metadata and unspecified addresses at dial time even
|
||||
* for a listed name;
|
||||
* - every call has a timeout, and every body is read with a byte cap.
|
||||
*
|
||||
* The origin handed in is the one the caller just re-checked against
|
||||
* `DOCUMENT_SOURCE_ORIGINS`. Nothing here decides trust; it only refuses to
|
||||
* go anywhere else.
|
||||
*/
|
||||
import { safeFetch, SafeFetchError } from "@/lib/safe-fetch";
|
||||
import { BodyTooLargeError, readBoundedBody } from "@/lib/labs/ocr-upload";
|
||||
import { annotate } from "@/lib/logging/context";
|
||||
|
||||
import type { DocumentSourceErrorCode } from "./types";
|
||||
|
||||
/** JSON bodies larger than this are not a search page; refuse them. */
|
||||
export const SOURCE_JSON_MAX_BYTES = 1024 * 1024;
|
||||
export const SOURCE_JSON_TIMEOUT_MS = 10_000;
|
||||
export const SOURCE_DOWNLOAD_TIMEOUT_MS = 60_000;
|
||||
|
||||
/**
|
||||
* A request to a source ended somewhere other than a usable answer. The code
|
||||
* is what the client translates; `upstreamStatus` is the only thing about the
|
||||
* source's answer that is passed on. Its body never is: an error page can
|
||||
* echo the request's credentials or internal paths.
|
||||
*/
|
||||
export class DocumentSourceError extends Error {
|
||||
readonly code: DocumentSourceErrorCode;
|
||||
readonly upstreamStatus?: number;
|
||||
|
||||
constructor(code: DocumentSourceErrorCode, upstreamStatus?: number) {
|
||||
super(`document source: ${code}`);
|
||||
this.name = "DocumentSourceError";
|
||||
this.code = code;
|
||||
if (upstreamStatus !== undefined) this.upstreamStatus = upstreamStatus;
|
||||
}
|
||||
}
|
||||
|
||||
/** The document is larger than the vault accepts; carries the cap. */
|
||||
export class DocumentSourceTooLargeError extends Error {
|
||||
constructor(readonly maxFileBytes: number) {
|
||||
super("document source: file larger than the vault accepts");
|
||||
this.name = "DocumentSourceTooLargeError";
|
||||
}
|
||||
}
|
||||
|
||||
export interface SourceTarget {
|
||||
/** The listed origin, exactly as `DOCUMENT_SOURCE_ORIGINS` holds it. */
|
||||
origin: string;
|
||||
/** Origin plus the optional base path, no trailing slash. */
|
||||
baseUrl: string;
|
||||
headers: Record<string, string>;
|
||||
/** How this source's error bodies read, when they say more than a status. */
|
||||
classify?: ErrorClassifier;
|
||||
}
|
||||
|
||||
async function sourceRequest(
|
||||
target: SourceTarget,
|
||||
path: string,
|
||||
timeoutMs: number,
|
||||
): Promise<Response> {
|
||||
const url = `${target.baseUrl}${path}`;
|
||||
let response: Response;
|
||||
try {
|
||||
response = await safeFetch(
|
||||
url,
|
||||
{ method: "GET", headers: target.headers, redirect: "manual" },
|
||||
{ operatorApprovedPrivateOrigin: target.origin, timeoutMs },
|
||||
);
|
||||
} catch (err) {
|
||||
// A refused destination, a timeout and a dead host all read the same to
|
||||
// the person: the source could not be reached from here. The operator's
|
||||
// wide event keeps the kind.
|
||||
annotate({
|
||||
meta: {
|
||||
sourceFailure: err instanceof SafeFetchError ? err.kind : "network",
|
||||
},
|
||||
});
|
||||
throw new DocumentSourceError("unreachable");
|
||||
}
|
||||
if (response.status >= 300 && response.status < 400) {
|
||||
await discard(response);
|
||||
throw new DocumentSourceError("redirected", response.status);
|
||||
}
|
||||
return response;
|
||||
}
|
||||
|
||||
async function discard(response: Response): Promise<void> {
|
||||
try {
|
||||
await response.body?.cancel();
|
||||
} catch {
|
||||
// nothing to release
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* A source-specific reading of an error answer: given the status and the
|
||||
* machine code in the body (if any), the error to report, or null for the
|
||||
* generic mapping. The body itself is never passed on.
|
||||
*/
|
||||
export type ErrorClassifier = (
|
||||
status: number,
|
||||
code: string | null,
|
||||
) => DocumentSourceErrorCode | null;
|
||||
|
||||
/** Longest error body read to find its machine code. */
|
||||
const ERROR_BODY_MAX_BYTES = 4 * 1024;
|
||||
|
||||
/** The `code` (or `error.code`) of a JSON error body, if there is one. */
|
||||
async function errorCodeOf(response: Response): Promise<string | null> {
|
||||
try {
|
||||
const bytes = await readBoundedBody(response.body, ERROR_BODY_MAX_BYTES);
|
||||
const body = JSON.parse(new TextDecoder().decode(bytes)) as unknown;
|
||||
if (!body || typeof body !== "object") return null;
|
||||
const record = body as Record<string, unknown>;
|
||||
const nested = record.error as Record<string, unknown> | undefined;
|
||||
const code =
|
||||
typeof record.code === "string"
|
||||
? record.code
|
||||
: nested &&
|
||||
typeof nested === "object" &&
|
||||
typeof nested.code === "string"
|
||||
? nested.code
|
||||
: null;
|
||||
return code && code.length <= 100 ? code : null;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/** Map a non-2xx answer to the error the person sees. */
|
||||
export async function failFor(
|
||||
response: Response,
|
||||
notFound: DocumentSourceErrorCode = "notFound",
|
||||
classify?: ErrorClassifier,
|
||||
): Promise<never> {
|
||||
const status = response.status;
|
||||
if (classify) {
|
||||
const specific = classify(status, await errorCodeOf(response));
|
||||
if (specific) throw new DocumentSourceError(specific, status);
|
||||
} else {
|
||||
await discard(response);
|
||||
}
|
||||
if (status === 401 || status === 403) {
|
||||
throw new DocumentSourceError("authRefused", status);
|
||||
}
|
||||
if (status === 404) throw new DocumentSourceError(notFound, status);
|
||||
// Paperless-ngx answers 406 to an API version it no longer (or not yet)
|
||||
// speaks.
|
||||
if (status === 406) throw new DocumentSourceError("versionTooOld", status);
|
||||
throw new DocumentSourceError("badResponse", status);
|
||||
}
|
||||
|
||||
/** GET a JSON document, bounded in time and size. */
|
||||
export async function sourceJson(
|
||||
target: SourceTarget,
|
||||
path: string,
|
||||
options: {
|
||||
notFound?: DocumentSourceErrorCode;
|
||||
classify?: ErrorClassifier;
|
||||
} = {},
|
||||
): Promise<{ body: unknown; response: Response }> {
|
||||
const response = await sourceRequest(target, path, SOURCE_JSON_TIMEOUT_MS);
|
||||
if (!response.ok) {
|
||||
return failFor(
|
||||
response,
|
||||
options.notFound,
|
||||
options.classify ?? target.classify,
|
||||
);
|
||||
}
|
||||
const declared = Number(response.headers.get("content-length") ?? 0);
|
||||
if (declared > SOURCE_JSON_MAX_BYTES) {
|
||||
await discard(response);
|
||||
throw new DocumentSourceError("badResponse", response.status);
|
||||
}
|
||||
let bytes: Uint8Array;
|
||||
try {
|
||||
bytes = await readBoundedBody(response.body, SOURCE_JSON_MAX_BYTES);
|
||||
} catch {
|
||||
throw new DocumentSourceError("badResponse", response.status);
|
||||
}
|
||||
try {
|
||||
return {
|
||||
body: JSON.parse(new TextDecoder().decode(bytes)) as unknown,
|
||||
response,
|
||||
};
|
||||
} catch {
|
||||
// A login page or a proxy's HTML error instead of the API: most often a
|
||||
// base address that points at the web app rather than its API root.
|
||||
throw new DocumentSourceError("badResponse", response.status);
|
||||
}
|
||||
}
|
||||
|
||||
/** GET a file, bounded by the vault's own per-file cap. */
|
||||
export async function sourceDownload(
|
||||
target: SourceTarget,
|
||||
path: string,
|
||||
maxFileBytes: number,
|
||||
): Promise<{ bytes: Uint8Array; filename: string | null }> {
|
||||
const response = await sourceRequest(
|
||||
target,
|
||||
path,
|
||||
SOURCE_DOWNLOAD_TIMEOUT_MS,
|
||||
);
|
||||
if (!response.ok) return failFor(response, "notFound", target.classify);
|
||||
const declared = Number(response.headers.get("content-length") ?? 0);
|
||||
if (declared > maxFileBytes) {
|
||||
await discard(response);
|
||||
throw new DocumentSourceTooLargeError(maxFileBytes);
|
||||
}
|
||||
try {
|
||||
const bytes = await readBoundedBody(response.body, maxFileBytes);
|
||||
return {
|
||||
bytes,
|
||||
filename: filenameFrom(response.headers.get("content-disposition")),
|
||||
};
|
||||
} catch (err) {
|
||||
if (err instanceof BodyTooLargeError) {
|
||||
throw new DocumentSourceTooLargeError(maxFileBytes);
|
||||
}
|
||||
throw new DocumentSourceError("unreachable");
|
||||
}
|
||||
}
|
||||
|
||||
/** The file name a download names, if it names one. */
|
||||
export function filenameFrom(header: string | null): string | null {
|
||||
if (!header) return null;
|
||||
const star = /filename\*=(?:UTF-8'')?([^;]+)/i.exec(header);
|
||||
if (star) {
|
||||
try {
|
||||
return decodeURIComponent(star[1].trim().replace(/^"|"$/g, ""));
|
||||
} catch {
|
||||
// fall through to the plain form
|
||||
}
|
||||
}
|
||||
const plain = /filename="?([^";]+)"?/i.exec(header);
|
||||
return plain ? plain[1] : null;
|
||||
}
|
||||
@@ -0,0 +1,95 @@
|
||||
/**
|
||||
* `DOCUMENT_SOURCE_ORIGINS` (#1038) — the operator grant that lets the
|
||||
* document picker reach a Paperless-ngx or Papra instance.
|
||||
*
|
||||
* DOCUMENT_SOURCE_ORIGINS="https://paperless.example.com,http://papra.lan:1221"
|
||||
*
|
||||
* The grammar is the one every private-origin list shares
|
||||
* (`src/lib/private-origin-policy.ts`): exact `scheme://host[:port]` origins,
|
||||
* no path, query, credentials, wildcard, CIDR or suffix. A malformed entry is
|
||||
* logged once, reduced to scheme and host, and grants nothing; the valid
|
||||
* entries beside it keep working.
|
||||
*
|
||||
* Unlike the notification list, this one is the WHOLE trust decision, not an
|
||||
* exception on top of a public-host default. An origin that is not listed is
|
||||
* refused even when it is a public address, and with the variable unset the
|
||||
* picker does not exist: the routes answer 404 and the client renders no
|
||||
* button. A person can only ever point their connection at a system the
|
||||
* operator named. Every request to a listed origin still goes through
|
||||
* `safeFetch` with the origin pinned, redirects forbidden, and the dial-time
|
||||
* floor that refuses link-local, metadata and unspecified addresses even for
|
||||
* a listed name.
|
||||
*
|
||||
* Read directly from `process.env` so the compose-whitelist guard sees it,
|
||||
* parsed once per distinct value, and re-read on every use: an operator who
|
||||
* removes an origin cuts off the stored connections that name it on their
|
||||
* next request.
|
||||
*/
|
||||
import {
|
||||
parsePrivateOrigins,
|
||||
redactGrantEntry,
|
||||
} from "@/lib/private-origin-policy";
|
||||
|
||||
const ENV_NAME = "DOCUMENT_SOURCE_ORIGINS";
|
||||
|
||||
let cachedRaw: string | undefined;
|
||||
let cachedOrigins: ReadonlySet<string> = new Set();
|
||||
|
||||
export function configuredDocumentSourceOrigins(): ReadonlySet<string> {
|
||||
const raw = process.env.DOCUMENT_SOURCE_ORIGINS;
|
||||
if (raw === cachedRaw) return cachedOrigins;
|
||||
cachedOrigins = parsePrivateOrigins(raw, (redactedEntry, reason) => {
|
||||
console.warn(`${ENV_NAME}: ignoring entry "${redactedEntry}": ${reason}`);
|
||||
});
|
||||
cachedRaw = raw;
|
||||
return cachedOrigins;
|
||||
}
|
||||
|
||||
/** Is the picker switched on for this instance at all? */
|
||||
export function documentSourcesAvailable(): boolean {
|
||||
return configuredDocumentSourceOrigins().size > 0;
|
||||
}
|
||||
|
||||
/** Longest base URL a connection may store. */
|
||||
export const DOCUMENT_SOURCE_BASE_URL_MAX = 2048;
|
||||
|
||||
export type SourceBaseVerdict =
|
||||
| { ok: true; origin: string; baseUrl: string }
|
||||
| { ok: false; reason: "invalid" | "notAllowed" };
|
||||
|
||||
/**
|
||||
* Validate a base URL a person typed against the operator's list.
|
||||
*
|
||||
* The base may carry a path (a Paperless served under `/paperless`); only its
|
||||
* origin is compared, and it must be listed exactly. Userinfo, a query and a
|
||||
* fragment are refused: none of them belongs in a base address, and each is
|
||||
* where a pasted credential would sit. Trailing slashes are trimmed so the
|
||||
* client can append `/api/...` without doubling one.
|
||||
*/
|
||||
export function evaluateSourceBaseUrl(
|
||||
value: string,
|
||||
origins: ReadonlySet<string> = configuredDocumentSourceOrigins(),
|
||||
): SourceBaseVerdict {
|
||||
let url: URL;
|
||||
try {
|
||||
url = new URL(value.trim());
|
||||
} catch {
|
||||
return { ok: false, reason: "invalid" };
|
||||
}
|
||||
if (url.protocol !== "http:" && url.protocol !== "https:") {
|
||||
return { ok: false, reason: "invalid" };
|
||||
}
|
||||
if (url.username || url.password || url.search || url.hash) {
|
||||
return { ok: false, reason: "invalid" };
|
||||
}
|
||||
if (!origins.has(url.origin)) return { ok: false, reason: "notAllowed" };
|
||||
const path = url.pathname.replace(/\/+$/, "");
|
||||
return { ok: true, origin: url.origin, baseUrl: `${url.origin}${path}` };
|
||||
}
|
||||
|
||||
/** The listed origins as the settings card may show them. */
|
||||
export function listedDocumentSourceOrigins(): string[] {
|
||||
return [...configuredDocumentSourceOrigins()]
|
||||
.map((origin) => redactGrantEntry(origin))
|
||||
.sort();
|
||||
}
|
||||
@@ -0,0 +1,165 @@
|
||||
/**
|
||||
* What every document-source route does before its own work, and how a source
|
||||
* failure becomes a response.
|
||||
*
|
||||
* The admission rule, with {@link admitDocumentSourceCaller} as its second
|
||||
* half:
|
||||
*
|
||||
* - the route's own `requireAuth()` with no scope: a session, or a `["*"]` token; a narrow
|
||||
* token is refused, and so is any request acting on somebody else's record
|
||||
* (403 `sharing.not_permitted`). A delegate, whatever their grant, and a
|
||||
* guardian acting for a managed record never reach a connection: it is the
|
||||
* owner's credential to a system outside HealthLog, and driving it would
|
||||
* read the owner's whole archive there, which no grant covers;
|
||||
* - then the transport must be the cookie. A `["*"]` Bearer (the native
|
||||
* client, a script holding a login token) is refused: the picker is a
|
||||
* browser surface, and a credential for another system is managed only
|
||||
* from one;
|
||||
* - the documents module must be on, and the operator must have listed at
|
||||
* least one origin in `DOCUMENT_SOURCE_ORIGINS`. Without the list the
|
||||
* routes answer 404 before reading a row.
|
||||
*/
|
||||
import type { AuthContext } from "@/lib/api-handler";
|
||||
import { apiError } from "@/lib/api-response";
|
||||
import { requireModuleEnabled } from "@/lib/modules/gate";
|
||||
import { checkRateLimit, type RateLimitResult } from "@/lib/rate-limit";
|
||||
|
||||
import { DocumentSourceError, DocumentSourceTooLargeError } from "./http";
|
||||
import { documentSourcesAvailable } from "./origins";
|
||||
import {
|
||||
documentSourceErrorCode,
|
||||
systemFromSlug,
|
||||
type DocumentPickerSystem,
|
||||
type DocumentSourceErrorCode,
|
||||
} from "./types";
|
||||
|
||||
/**
|
||||
* Admit a caller the route has already resolved with a bare `requireAuth()`.
|
||||
*
|
||||
* Each route calls `requireAuth()` itself, in its own module, so the sharing
|
||||
* guard's scan sees every one of them as a bare, owner-only route rather than
|
||||
* a helper it cannot follow. This adds the rest: a cookie transport, the
|
||||
* documents module and the operator's list. The status read and the delete
|
||||
* pass `needsList: false, needsModule: false`: they touch only the person's
|
||||
* own row and never the network.
|
||||
*/
|
||||
export async function admitDocumentSourceCaller(
|
||||
auth: AuthContext,
|
||||
options: { needsList?: boolean; needsModule?: boolean } = {},
|
||||
): Promise<Response | null> {
|
||||
if (auth.authMethod !== "cookie") {
|
||||
return apiError(
|
||||
"Document archives are managed from a browser session.",
|
||||
403,
|
||||
{ errorCode: "documents.sources.browserOnly" },
|
||||
);
|
||||
}
|
||||
// Reading and removing one's own connection needs neither the module nor
|
||||
// the list: a person who switched Documents off, or whose operator removed
|
||||
// the list, must still be able to see and delete the stored token.
|
||||
if (options.needsModule !== false) {
|
||||
const gate = await requireModuleEnabled(auth.user.id, "inboundDocuments");
|
||||
if (!gate.enabled) return gate.response;
|
||||
}
|
||||
if (options.needsList !== false && !documentSourcesAvailable()) {
|
||||
return sourceErrorResponse("unavailable");
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/** The `{system}` path segment, or a 404 for anything else. */
|
||||
export async function systemParam(
|
||||
params: Promise<{ system: string }>,
|
||||
): Promise<DocumentPickerSystem | Response> {
|
||||
const { system } = await params;
|
||||
const resolved = systemFromSlug(system);
|
||||
return resolved ?? apiError("Unknown document system.", 404);
|
||||
}
|
||||
|
||||
const STATUS: Record<DocumentSourceErrorCode, number> = {
|
||||
unavailable: 404,
|
||||
notConnected: 404,
|
||||
originNotAllowed: 422,
|
||||
unreachable: 502,
|
||||
redirected: 502,
|
||||
authRefused: 502,
|
||||
permissionMissing: 502,
|
||||
versionTooOld: 502,
|
||||
badResponse: 502,
|
||||
notFound: 404,
|
||||
linkTargetNotFound: 404,
|
||||
rateLimited: 429,
|
||||
};
|
||||
|
||||
const MESSAGE: Record<DocumentSourceErrorCode, string> = {
|
||||
unavailable: "Document archives are not enabled on this server.",
|
||||
notConnected: "This document archive is not connected.",
|
||||
originNotAllowed:
|
||||
"This address is not on the server's list of allowed document archives.",
|
||||
unreachable: "The document archive could not be reached.",
|
||||
redirected:
|
||||
"The document archive redirected the request. Use the address it redirects to.",
|
||||
authRefused: "The document archive refused the API token.",
|
||||
permissionMissing:
|
||||
"The API key cannot read the archive's tags. Give it the tags:read permission.",
|
||||
versionTooOld: "The document archive's version is too old.",
|
||||
badResponse: "The document archive gave an answer HealthLog cannot read.",
|
||||
notFound: "The document archive has no such document or organization.",
|
||||
linkTargetNotFound: "The record to link to was not found.",
|
||||
rateLimited: "Too many requests. Try again later.",
|
||||
};
|
||||
|
||||
export function sourceErrorResponse(
|
||||
code: DocumentSourceErrorCode,
|
||||
upstreamStatus?: number,
|
||||
headers?: Record<string, string>,
|
||||
): Response {
|
||||
return apiError(MESSAGE[code], STATUS[code], {
|
||||
errorCode: documentSourceErrorCode(code),
|
||||
...(upstreamStatus !== undefined ? { upstreamStatus } : {}),
|
||||
...(headers ? { headers } : {}),
|
||||
});
|
||||
}
|
||||
|
||||
/** Map a thrown source failure to its response; rethrow anything else. */
|
||||
export function responseForSourceFailure(err: unknown): Response {
|
||||
if (err instanceof DocumentSourceError) {
|
||||
return sourceErrorResponse(err.code, err.upstreamStatus);
|
||||
}
|
||||
if (err instanceof DocumentSourceTooLargeError) {
|
||||
return apiError("File is too large.", 413, {
|
||||
errorCode: "documents.inbound.fileTooLarge",
|
||||
reason: "fileTooLarge",
|
||||
maxFileBytes: err.maxFileBytes,
|
||||
});
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
|
||||
/** Saves and tests share one allowance: each one dials the source. */
|
||||
const CONNECT_LIMIT_PER_MINUTE = 10;
|
||||
|
||||
export function checkConnectRateLimit(
|
||||
userId: string,
|
||||
): Promise<RateLimitResult> {
|
||||
return checkRateLimit(
|
||||
`documents-source-connect:${userId}`,
|
||||
CONNECT_LIMIT_PER_MINUTE,
|
||||
60_000,
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Searches and tag reads per minute. Each one is a request to the source; a
|
||||
* debounced search field sends one per pause in typing, which this leaves
|
||||
* ample room for.
|
||||
*/
|
||||
const SEARCH_LIMIT_PER_MINUTE = 60;
|
||||
|
||||
export function checkSearchRateLimit(userId: string): Promise<RateLimitResult> {
|
||||
return checkRateLimit(
|
||||
`documents-source-search:${userId}`,
|
||||
SEARCH_LIMIT_PER_MINUTE,
|
||||
60_000,
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,157 @@
|
||||
/**
|
||||
* Shared vocabulary of the document picker (#1038): which systems it speaks,
|
||||
* the minimal shapes it hands to the client, and the errors a source can end
|
||||
* in. A zero-dependency leaf so the client components, the OpenAPI registry
|
||||
* and the routes import one list.
|
||||
*/
|
||||
|
||||
/** The systems the picker can search and import from. */
|
||||
export const DOCUMENT_PICKER_SYSTEMS = ["PAPERLESS", "PAPRA"] as const;
|
||||
export type DocumentPickerSystem = (typeof DOCUMENT_PICKER_SYSTEMS)[number];
|
||||
|
||||
/** The lowercase path segment for each system (`/api/documents/sources/{slug}`). */
|
||||
export const DOCUMENT_PICKER_SLUGS = {
|
||||
paperless: "PAPERLESS",
|
||||
papra: "PAPRA",
|
||||
} as const satisfies Record<string, DocumentPickerSystem>;
|
||||
export type DocumentPickerSlug = keyof typeof DOCUMENT_PICKER_SLUGS;
|
||||
|
||||
export function systemFromSlug(
|
||||
slug: string | undefined,
|
||||
): DocumentPickerSystem | null {
|
||||
return slug && Object.hasOwn(DOCUMENT_PICKER_SLUGS, slug)
|
||||
? DOCUMENT_PICKER_SLUGS[slug as DocumentPickerSlug]
|
||||
: null;
|
||||
}
|
||||
|
||||
export function slugForSystem(
|
||||
system: DocumentPickerSystem,
|
||||
): DocumentPickerSlug {
|
||||
return system === "PAPERLESS" ? "paperless" : "papra";
|
||||
}
|
||||
|
||||
/** Results per search page. */
|
||||
export const DOCUMENT_PICKER_PAGE_SIZE = 25;
|
||||
/** Highest page a search may ask for. */
|
||||
export const DOCUMENT_PICKER_MAX_PAGE = 500;
|
||||
/** Longest name query. */
|
||||
export const DOCUMENT_PICKER_QUERY_MAX = 200;
|
||||
/** Documents the sheet imports in one run. */
|
||||
export const DOCUMENT_PICKER_MAX_SELECTION = 25;
|
||||
/** Tags offered in the filter (read in pages of 250). */
|
||||
export const DOCUMENT_PICKER_MAX_TAGS = 1000;
|
||||
|
||||
/** Paperless-ngx document ids are integers. */
|
||||
export const PAPERLESS_ID = /^\d{1,18}$/;
|
||||
/** Papra document ids: letters, digits, `_` and `-`. */
|
||||
export const PAPRA_ID = /^[A-Za-z0-9_-]{1,128}$/;
|
||||
/** Papra organization ids, as Papra issues them. */
|
||||
export const PAPRA_ORG_ID = /^org_[a-z0-9]{24}$/;
|
||||
|
||||
/**
|
||||
* Is `id` a document id `system` could have issued? Checked before an id is
|
||||
* put into a request path, so nothing a request carries can add a path
|
||||
* segment, a query or a `..` to the URL built from it.
|
||||
*/
|
||||
export function isSourceDocumentId(
|
||||
system: DocumentPickerSystem,
|
||||
id: string,
|
||||
): boolean {
|
||||
if (id === "." || id === "..") return false;
|
||||
return (system === "PAPERLESS" ? PAPERLESS_ID : PAPRA_ID).test(id);
|
||||
}
|
||||
|
||||
/** A connection as every route may show it: never the token. */
|
||||
export interface DocumentSourceConnectionDto {
|
||||
system: DocumentPickerSystem;
|
||||
baseUrl: string;
|
||||
organizationId: string | null;
|
||||
hasToken: true;
|
||||
lastVerifiedAt: string | null;
|
||||
/** False once the operator removed the origin from the list. */
|
||||
originAllowed: boolean;
|
||||
}
|
||||
|
||||
export interface DocumentSourcesStatusDto {
|
||||
/** The operator listed at least one origin; otherwise the picker is off. */
|
||||
available: boolean;
|
||||
/** The listed origins, so the settings card can say what is allowed. */
|
||||
allowedOrigins: string[];
|
||||
connections: DocumentSourceConnectionDto[];
|
||||
}
|
||||
|
||||
export interface DocumentSourceTagDto {
|
||||
id: string;
|
||||
name: string;
|
||||
}
|
||||
|
||||
/** Whether HealthLog already holds a result, by its source key. */
|
||||
export type DocumentPickerResultState = "new" | "imported" | "deleted";
|
||||
|
||||
export interface DocumentSourceResultDto {
|
||||
sourceId: string;
|
||||
title: string;
|
||||
/** YYYY-MM-DD, the document's own date in the source, or null. */
|
||||
date: string | null;
|
||||
tags: string[];
|
||||
sizeBytes: number | null;
|
||||
state: DocumentPickerResultState;
|
||||
/** The vault document when `state` is `imported`. */
|
||||
documentId: string | null;
|
||||
}
|
||||
|
||||
export interface DocumentSourceSearchDto {
|
||||
results: DocumentSourceResultDto[];
|
||||
page: number;
|
||||
hasMore: boolean;
|
||||
}
|
||||
|
||||
export type DocumentImportOutcome = "imported" | "duplicate" | "deleted";
|
||||
|
||||
export interface DocumentSourceImportDto {
|
||||
outcome: DocumentImportOutcome;
|
||||
/** The vault document; null only when a deleted one has been purged. */
|
||||
documentId: string | null;
|
||||
/** The document was linked to the record the picker was opened from. */
|
||||
linked: boolean;
|
||||
}
|
||||
|
||||
/** Record kinds a picked document can be linked to on import. */
|
||||
export const DOCUMENT_PICKER_LINK_KINDS = [
|
||||
"conditionEpisode",
|
||||
"encounter",
|
||||
"vaccination",
|
||||
] as const;
|
||||
export type DocumentPickerLinkKind =
|
||||
(typeof DOCUMENT_PICKER_LINK_KINDS)[number];
|
||||
|
||||
/**
|
||||
* Why a request to a source failed, as the `meta.errorCode` values the routes
|
||||
* emit. The route helper keys its status and wording on the last segment.
|
||||
*/
|
||||
export const DOCUMENT_SOURCE_ERROR_CODES = [
|
||||
"documents.sources.unavailable",
|
||||
"documents.sources.notConnected",
|
||||
"documents.sources.originNotAllowed",
|
||||
"documents.sources.unreachable",
|
||||
"documents.sources.redirected",
|
||||
"documents.sources.authRefused",
|
||||
"documents.sources.permissionMissing",
|
||||
"documents.sources.versionTooOld",
|
||||
"documents.sources.badResponse",
|
||||
"documents.sources.notFound",
|
||||
"documents.sources.linkTargetNotFound",
|
||||
"documents.sources.rateLimited",
|
||||
] as const;
|
||||
type SourceErrorSuffix<T> = T extends `documents.sources.${infer S}`
|
||||
? S
|
||||
: never;
|
||||
export type DocumentSourceErrorCode = SourceErrorSuffix<
|
||||
(typeof DOCUMENT_SOURCE_ERROR_CODES)[number]
|
||||
>;
|
||||
|
||||
export function documentSourceErrorCode(
|
||||
code: DocumentSourceErrorCode,
|
||||
): (typeof DOCUMENT_SOURCE_ERROR_CODES)[number] {
|
||||
return `documents.sources.${code}`;
|
||||
}
|
||||
@@ -620,6 +620,8 @@ export const NOT_IN_BACKUP_MODELS: Readonly<Record<string, string>> = {
|
||||
"A CSRF nonce for a handshake that is valid for minutes and is already over by the time any backup is read.",
|
||||
McpOAuthConnection:
|
||||
"A grant issued to a client that was registered against this instance.",
|
||||
DocumentSourceConnection:
|
||||
"A person's API token for their Paperless-ngx or Papra instance. A backup file is not a credential store, same reasoning as ApiToken; and restored on another host it would point at an address that host's operator never allowed in DOCUMENT_SOURCE_ORIGINS. The documents imported through it are carried with the vault; reconnecting takes one form.",
|
||||
IntegrationStatus:
|
||||
"A ledger of what this deployment's syncs did. Restoring it onto another host would assert a sync history that host never had.",
|
||||
|
||||
|
||||
@@ -177,6 +177,8 @@ export interface DocumentBackupEntry {
|
||||
*/
|
||||
sourceSystem?: string | null;
|
||||
sourceId?: string | null;
|
||||
/** v1.39.3 — the source instance (origin); absent in older files. */
|
||||
sourceInstance?: string | null;
|
||||
/** Canonical DR only: the import's hold on automatic AI reading. */
|
||||
aiReadDeferred?: boolean;
|
||||
}
|
||||
@@ -309,6 +311,7 @@ export async function buildRecordsBackupSection(
|
||||
lastIndexOutcome: true,
|
||||
sourceSystem: true,
|
||||
sourceId: true,
|
||||
sourceInstance: true,
|
||||
aiReadDeferred: true,
|
||||
createdAt: true,
|
||||
updatedAt: true,
|
||||
@@ -330,6 +333,7 @@ export async function buildRecordsBackupSection(
|
||||
summaryEncrypted: true,
|
||||
sourceSystem: true,
|
||||
sourceId: true,
|
||||
sourceInstance: true,
|
||||
createdAt: true,
|
||||
},
|
||||
});
|
||||
@@ -545,6 +549,7 @@ export async function buildRecordsBackupSection(
|
||||
: null,
|
||||
sourceSystem: d.sourceSystem,
|
||||
sourceId: d.sourceId,
|
||||
sourceInstance: d.sourceInstance,
|
||||
createdAt: d.createdAt.toISOString(),
|
||||
};
|
||||
|
||||
|
||||
@@ -30,6 +30,7 @@
|
||||
import { Buffer } from "node:buffer";
|
||||
|
||||
import { prisma, toJson } from "@/lib/db";
|
||||
import { normaliseSourceInstance } from "@/lib/validations/inbound-documents";
|
||||
import { auditLog } from "@/lib/auth/audit";
|
||||
import {
|
||||
isStoredBackupReadError,
|
||||
@@ -2057,6 +2058,14 @@ export async function restoreBackup(
|
||||
sourceId: document.sourceSystem
|
||||
? (document.sourceId ?? null)
|
||||
: null,
|
||||
// A file written before v1.39.3 has no instance: the key then
|
||||
// matches any instance of its system, as it did when stored.
|
||||
sourceInstance:
|
||||
document.sourceSystem && document.sourceId
|
||||
? document.sourceInstance
|
||||
? normaliseSourceInstance(document.sourceInstance)
|
||||
: null
|
||||
: null,
|
||||
createdAt: new Date(document.createdAt!),
|
||||
updatedAt: new Date(document.updatedAt!),
|
||||
})),
|
||||
|
||||
@@ -58,9 +58,12 @@ export async function purgeExpiredDocumentTombstones(
|
||||
userId: true,
|
||||
sourceSystem: true,
|
||||
sourceId: true,
|
||||
sourceInstance: true,
|
||||
// Further keys the import answered with this document (same bytes).
|
||||
// They cascade away with the row, so they are copied here first.
|
||||
sourceAliases: { select: { sourceSystem: true, sourceId: true } },
|
||||
sourceAliases: {
|
||||
select: { sourceSystem: true, sourceId: true, sourceInstance: true },
|
||||
},
|
||||
},
|
||||
});
|
||||
const keys = keyed.flatMap((row) => [
|
||||
@@ -70,6 +73,7 @@ export async function purgeExpiredDocumentTombstones(
|
||||
userId: row.userId,
|
||||
sourceSystem: row.sourceSystem,
|
||||
sourceId: row.sourceId,
|
||||
sourceInstance: row.sourceInstance,
|
||||
},
|
||||
]
|
||||
: []),
|
||||
@@ -77,6 +81,7 @@ export async function purgeExpiredDocumentTombstones(
|
||||
userId: row.userId,
|
||||
sourceSystem: alias.sourceSystem,
|
||||
sourceId: alias.sourceId,
|
||||
sourceInstance: alias.sourceInstance,
|
||||
})),
|
||||
]);
|
||||
if (keys.length > 0) {
|
||||
|
||||
@@ -136,6 +136,22 @@ export const ERROR_CODE_CATALOGUE: Readonly<Record<string, readonly string[]>> =
|
||||
"documents.inbound.uploadBusy",
|
||||
"documents.inbound.uploadTimeout",
|
||||
"documents.inbound.vaccinationNotFound",
|
||||
"documents.sources.authRefused",
|
||||
"documents.sources.badResponse",
|
||||
"documents.sources.browserOnly",
|
||||
"documents.sources.invalidAddress",
|
||||
"documents.sources.linkTargetNotFound",
|
||||
"documents.sources.notConnected",
|
||||
"documents.sources.notFound",
|
||||
"documents.sources.organizationRequired",
|
||||
"documents.sources.originNotAllowed",
|
||||
"documents.sources.permissionMissing",
|
||||
"documents.sources.rateLimited",
|
||||
"documents.sources.redirected",
|
||||
"documents.sources.tokenRequired",
|
||||
"documents.sources.unavailable",
|
||||
"documents.sources.unreachable",
|
||||
"documents.sources.versionTooOld",
|
||||
],
|
||||
encounter: [
|
||||
"encounter.invalid",
|
||||
|
||||
@@ -0,0 +1,402 @@
|
||||
/**
|
||||
* OpenAPI route table for the document picker (`/api/documents/sources/*`,
|
||||
* #1038): a person's connection to Paperless-ngx or Papra, searching it, and
|
||||
* importing picked documents into the vault.
|
||||
*
|
||||
* Every operation is cookie-only (declared per operation as `cookieAuth`): a
|
||||
* Bearer, even a `["*"]` one, is refused 403 `documents.sources.browserOnly`,
|
||||
* and so is a request acting on another person's record. The native client does
|
||||
* not use these routes. Request bodies reuse the runtime Zod schemas.
|
||||
*/
|
||||
import type { ZodOpenApiObject } from "zod-openapi";
|
||||
import { z } from "zod/v4";
|
||||
|
||||
import {
|
||||
DOCUMENT_PICKER_LINK_KINDS,
|
||||
DOCUMENT_PICKER_MAX_PAGE,
|
||||
DOCUMENT_PICKER_QUERY_MAX,
|
||||
DOCUMENT_PICKER_SYSTEMS,
|
||||
} from "@/lib/documents/sources/types";
|
||||
import {
|
||||
documentSourceImportSchema,
|
||||
documentSourceSaveSchema,
|
||||
} from "@/lib/validations/document-sources";
|
||||
|
||||
import { dataEnvelope, errorEnvelope, stdResponses } from "./shared";
|
||||
|
||||
const cookieOnly = [{ cookieAuth: [] }];
|
||||
|
||||
const systemParameter = {
|
||||
name: "system",
|
||||
in: "path" as const,
|
||||
required: true,
|
||||
description: "`paperless` (Paperless-ngx) or `papra` (Papra).",
|
||||
schema: { type: "string" as const, enum: ["paperless", "papra"] },
|
||||
};
|
||||
|
||||
const systemEnum = z.enum(DOCUMENT_PICKER_SYSTEMS);
|
||||
|
||||
const connectionSchema = z
|
||||
.object({
|
||||
system: systemEnum,
|
||||
baseUrl: z.string(),
|
||||
organizationId: z.string().nullable(),
|
||||
hasToken: z.literal(true),
|
||||
lastVerifiedAt: z.string().nullable(),
|
||||
originAllowed: z.boolean(),
|
||||
})
|
||||
.meta({ id: "DocumentSourceConnection" });
|
||||
|
||||
const statusSchema = z
|
||||
.object({
|
||||
available: z.boolean(),
|
||||
allowedOrigins: z.array(z.string()),
|
||||
connections: z.array(connectionSchema),
|
||||
})
|
||||
.meta({ id: "DocumentSourcesStatus" });
|
||||
|
||||
const tagSchema = z
|
||||
.object({ id: z.string(), name: z.string() })
|
||||
.meta({ id: "DocumentSourceTag" });
|
||||
|
||||
const resultSchema = z
|
||||
.object({
|
||||
sourceId: z.string(),
|
||||
title: z.string(),
|
||||
date: z.string().nullable(),
|
||||
tags: z.array(z.string()),
|
||||
sizeBytes: z.number().int().nullable(),
|
||||
state: z.enum(["new", "imported", "deleted"]),
|
||||
documentId: z.string().nullable(),
|
||||
})
|
||||
.meta({ id: "DocumentSourceResult" });
|
||||
|
||||
const searchSchema = z
|
||||
.object({
|
||||
results: z.array(resultSchema),
|
||||
page: z.number().int(),
|
||||
hasMore: z.boolean(),
|
||||
})
|
||||
.meta({ id: "DocumentSourceSearch" });
|
||||
|
||||
const importResultSchema = z
|
||||
.object({
|
||||
outcome: z.enum(["imported", "duplicate", "deleted"]),
|
||||
documentId: z.string().nullable(),
|
||||
linked: z.boolean(),
|
||||
})
|
||||
.meta({ id: "DocumentSourceImport" });
|
||||
|
||||
const refusal = {
|
||||
description:
|
||||
"Refused before any request to the source: `documents.sources.browserOnly` for a Bearer caller, `sharing.not_permitted` while acting on another person's record (a delegate or guardian never uses the owner's archive), or `module.disabled` when the Documents module is off.",
|
||||
content: { "application/json": { schema: errorEnvelope } },
|
||||
};
|
||||
|
||||
const unavailable = {
|
||||
description:
|
||||
"`documents.sources.unavailable`: the operator has not listed any origin in `DOCUMENT_SOURCE_ORIGINS`, so the picker is off. `documents.sources.notConnected`: no connection is saved for this system. `documents.sources.notFound`: the source has no such document, or (Papra) no such organization, including one the key's user is not a member of (Papra 403 `user.not_in_organization`) and one Papra cannot parse (400).",
|
||||
content: { "application/json": { schema: errorEnvelope } },
|
||||
};
|
||||
|
||||
const upstream = {
|
||||
description:
|
||||
"The source could not answer usefully, with the reason in `meta.errorCode` and, when the source answered at all, its HTTP status in `meta.upstreamStatus` (its body is never passed on): `documents.sources.unreachable` (no connection, timeout, or a destination refused at dial time), `documents.sources.redirected` (the source answered with a redirect, which is never followed), `documents.sources.authRefused` (401/403 from the source), `documents.sources.permissionMissing` (Papra: the key cannot read tags; needs `tags:read`), `documents.sources.versionTooOld` (Paperless-ngx older than 2.16 / API version 9), `documents.sources.badResponse` (not the documented JSON, or larger than 1 MiB).",
|
||||
content: { "application/json": { schema: errorEnvelope } },
|
||||
};
|
||||
|
||||
const originRefusal = {
|
||||
description:
|
||||
"`documents.sources.originNotAllowed`: the base address's origin is not on the operator's list (checked on every request, so a connection saved earlier is cut off once its origin is removed), or another validation failure.",
|
||||
content: { "application/json": { schema: errorEnvelope } },
|
||||
};
|
||||
|
||||
export const documentSourcePaths: NonNullable<ZodOpenApiObject["paths"]> = {
|
||||
"/api/documents/sources": {
|
||||
get: {
|
||||
tags: ["Documents"],
|
||||
summary: "Document archives: availability and connections",
|
||||
description:
|
||||
"Whether the operator enabled the document picker (`available`, true when `DOCUMENT_SOURCE_ORIGINS` lists at least one origin), the listed origins, and the caller's own saved connections to Paperless-ngx or Papra. A connection never carries its token: `hasToken` is always true for a saved one, and no route returns the token in whole or in part. `originAllowed` is false once the operator removed the connection's origin from the list. No request leaves the server. Needs neither the Documents module nor the list: with the list unset, `available` is false and any connection still stored is listed (with `originAllowed: false`) so it can be removed. Cookie session only; not delegable.",
|
||||
security: cookieOnly,
|
||||
responses: {
|
||||
"200": {
|
||||
description: "The picker's state for the caller.",
|
||||
content: {
|
||||
"application/json": {
|
||||
schema: dataEnvelope(
|
||||
statusSchema,
|
||||
"DocumentSourcesStatusEnvelope",
|
||||
),
|
||||
},
|
||||
},
|
||||
},
|
||||
"403": refusal,
|
||||
...stdResponses,
|
||||
},
|
||||
},
|
||||
},
|
||||
"/api/documents/sources/{system}": {
|
||||
put: {
|
||||
tags: ["Documents"],
|
||||
summary: "Save a connection to a document archive",
|
||||
description:
|
||||
"Saves or replaces the caller's connection to Paperless-ngx (`paperless`: base address and API token) or Papra (`papra`: base address, organization id, API key with `documents:read` and `tags:read`). The base address may carry a path; its origin must be listed in `DOCUMENT_SOURCE_ORIGINS`. The connection is tested live before anything is stored. Leaving out `token` keeps the stored one, but only when the new address is on the same origin as the saved one: the first save and a move to another origin must carry it (422 `documents.sources.tokenRequired`), so a stored token is never sent to an address it was not saved for. A Papra organization id is letters, digits, `_` and `-`. The token is stored encrypted (AES-256-GCM) and never returned. Ten saves and tests a minute per person. Cookie session only; not delegable.",
|
||||
security: cookieOnly,
|
||||
parameters: [systemParameter],
|
||||
requestBody: {
|
||||
required: true,
|
||||
content: { "application/json": { schema: documentSourceSaveSchema } },
|
||||
},
|
||||
responses: {
|
||||
"200": {
|
||||
description: "Saved and verified. The connection, without its token.",
|
||||
content: {
|
||||
"application/json": {
|
||||
schema: dataEnvelope(
|
||||
connectionSchema,
|
||||
"DocumentSourceConnectionEnvelope",
|
||||
),
|
||||
},
|
||||
},
|
||||
},
|
||||
"403": refusal,
|
||||
"404": unavailable,
|
||||
"422": {
|
||||
description:
|
||||
"`documents.sources.originNotAllowed`, `documents.sources.invalidAddress`, `documents.sources.organizationRequired` (Papra), `documents.sources.tokenRequired` (first save without a token), or body validation.",
|
||||
content: { "application/json": { schema: errorEnvelope } },
|
||||
},
|
||||
"502": upstream,
|
||||
"401": stdResponses["401"],
|
||||
"429": stdResponses["429"],
|
||||
},
|
||||
},
|
||||
delete: {
|
||||
tags: ["Documents"],
|
||||
summary: "Remove a connection to a document archive",
|
||||
description:
|
||||
"Deletes the caller's connection for this system, and its token with it. Works even after the operator removed the origin from the list, and with the Documents module switched off. `disconnected` is false when there was nothing to delete. Cookie session only; not delegable.",
|
||||
security: cookieOnly,
|
||||
parameters: [systemParameter],
|
||||
responses: {
|
||||
"200": {
|
||||
description: "Removed, or there was nothing to remove.",
|
||||
content: {
|
||||
"application/json": {
|
||||
schema: dataEnvelope(
|
||||
z
|
||||
.object({ disconnected: z.boolean() })
|
||||
.meta({ id: "DocumentSourceDisconnect" }),
|
||||
"DocumentSourceDisconnectEnvelope",
|
||||
),
|
||||
},
|
||||
},
|
||||
},
|
||||
"403": refusal,
|
||||
...stdResponses,
|
||||
},
|
||||
},
|
||||
},
|
||||
"/api/documents/sources/{system}/test": {
|
||||
post: {
|
||||
tags: ["Documents"],
|
||||
summary: "Test a saved connection",
|
||||
description:
|
||||
"One request to the source with the stored token. Answers `ok` with the round-trip time and stamps `lastVerifiedAt`, or the reason in `meta.errorCode`. Shares the save's allowance of ten a minute. Cookie session only; not delegable.",
|
||||
security: cookieOnly,
|
||||
parameters: [systemParameter],
|
||||
responses: {
|
||||
"200": {
|
||||
description: "The source answered.",
|
||||
content: {
|
||||
"application/json": {
|
||||
schema: dataEnvelope(
|
||||
z
|
||||
.object({
|
||||
ok: z.literal(true),
|
||||
latencyMs: z.number().int(),
|
||||
lastVerifiedAt: z.string(),
|
||||
})
|
||||
.meta({ id: "DocumentSourceTest" }),
|
||||
"DocumentSourceTestEnvelope",
|
||||
),
|
||||
},
|
||||
},
|
||||
},
|
||||
"403": refusal,
|
||||
"404": unavailable,
|
||||
"422": originRefusal,
|
||||
"502": upstream,
|
||||
"401": stdResponses["401"],
|
||||
"429": stdResponses["429"],
|
||||
},
|
||||
},
|
||||
},
|
||||
"/api/documents/sources/{system}/tags": {
|
||||
get: {
|
||||
tags: ["Documents"],
|
||||
summary: "Tags of a connected archive",
|
||||
description:
|
||||
"The source's tags for the picker's filter, the first 250 by name, in the source's own ids. Shares the search allowance of sixty a minute. Cookie session only; not delegable.",
|
||||
security: cookieOnly,
|
||||
parameters: [systemParameter],
|
||||
responses: {
|
||||
"200": {
|
||||
description: "The tags.",
|
||||
content: {
|
||||
"application/json": {
|
||||
schema: dataEnvelope(
|
||||
z
|
||||
.object({ tags: z.array(tagSchema) })
|
||||
.meta({ id: "DocumentSourceTags" }),
|
||||
"DocumentSourceTagsEnvelope",
|
||||
),
|
||||
},
|
||||
},
|
||||
},
|
||||
"403": refusal,
|
||||
"404": unavailable,
|
||||
"422": originRefusal,
|
||||
"502": upstream,
|
||||
"401": stdResponses["401"],
|
||||
"429": stdResponses["429"],
|
||||
},
|
||||
},
|
||||
},
|
||||
"/api/documents/sources/{system}/search": {
|
||||
get: {
|
||||
tags: ["Documents"],
|
||||
summary: "Search a connected archive by name, tag and date",
|
||||
description:
|
||||
"Searches the source 25 documents at a time. `q` matches the document's name (Paperless-ngx title, Papra name); `tag` narrows to one of the source's tags by id; `from` / `to` to a date range, inclusive (Paperless-ngx filters on `created__gte` / `created__lte`; Papra's range is applied to each page after it arrives). When a filter applied after arrival empties a page, the route reads on, up to five source pages, and `page` in the answer is the last source page read; ask for `page + 1` next. Each result carries what the vault already holds under that source key: `new`, `imported` with the vault document's `documentId`, or `deleted` (deleted in HealthLog; importing it again stores nothing). Nothing is stored. Sixty searches a minute, shared with the tag read. Cookie session only; not delegable.",
|
||||
security: cookieOnly,
|
||||
parameters: [
|
||||
systemParameter,
|
||||
{
|
||||
name: "q",
|
||||
in: "query",
|
||||
required: false,
|
||||
schema: { type: "string", maxLength: DOCUMENT_PICKER_QUERY_MAX },
|
||||
},
|
||||
{
|
||||
name: "tag",
|
||||
in: "query",
|
||||
required: false,
|
||||
schema: { type: "string", maxLength: 64 },
|
||||
},
|
||||
{
|
||||
name: "from",
|
||||
in: "query",
|
||||
required: false,
|
||||
schema: { type: "string", format: "date" },
|
||||
},
|
||||
{
|
||||
name: "to",
|
||||
in: "query",
|
||||
required: false,
|
||||
schema: { type: "string", format: "date" },
|
||||
},
|
||||
{
|
||||
name: "page",
|
||||
in: "query",
|
||||
required: false,
|
||||
schema: {
|
||||
type: "integer",
|
||||
minimum: 1,
|
||||
maximum: DOCUMENT_PICKER_MAX_PAGE,
|
||||
default: 1,
|
||||
},
|
||||
},
|
||||
],
|
||||
responses: {
|
||||
"200": {
|
||||
description: "One page of results.",
|
||||
content: {
|
||||
"application/json": {
|
||||
schema: dataEnvelope(
|
||||
searchSchema,
|
||||
"DocumentSourceSearchEnvelope",
|
||||
),
|
||||
},
|
||||
},
|
||||
},
|
||||
"403": refusal,
|
||||
"404": unavailable,
|
||||
"422": originRefusal,
|
||||
"502": upstream,
|
||||
"401": stdResponses["401"],
|
||||
"429": stdResponses["429"],
|
||||
},
|
||||
},
|
||||
},
|
||||
"/api/documents/sources/{system}/import": {
|
||||
post: {
|
||||
tags: ["Documents"],
|
||||
summary: "Import one picked document",
|
||||
description:
|
||||
"`sourceId` must be an id the system issues (Paperless-ngx: digits; Papra: letters, digits, `_`, `-`), else 422 before anything is fetched. Fetches one document from the source and stores it in the vault through the same path as an upload, with `sourceSystem` / `sourceId` / `sourceInstance` (the connection's origin) provenance; the same id from another instance is another document. One import per key runs at a time; a concurrent second one waits and then answers `duplicate`. The key check counts toward the lookup allowance, the fetch toward the search allowance, and only a file that arrived toward the upload allowance. A source key the vault already holds is answered without a download: `duplicate` for a live document (same id), `deleted` for one the person deleted in HealthLog, which is never stored again. Bytes already in the vault under another key are `duplicate` too, and the key is remembered for that document. `link` files the document against the record the picker was opened from (a condition, a visit or a vaccination, which must be the caller's own and live; checked before anything is fetched), including a duplicate. `kind` sets the document type (default OTHER). Automatic AI reading follows the person's own setting. Counts against the person's upload allowance (60 an hour) and storage quota. 201 for a new document, 200 otherwise. Cookie session only; not delegable.",
|
||||
security: cookieOnly,
|
||||
parameters: [systemParameter],
|
||||
requestBody: {
|
||||
required: true,
|
||||
content: {
|
||||
"application/json": { schema: documentSourceImportSchema },
|
||||
},
|
||||
},
|
||||
responses: {
|
||||
"201": {
|
||||
description: "Imported.",
|
||||
content: {
|
||||
"application/json": {
|
||||
schema: dataEnvelope(
|
||||
importResultSchema,
|
||||
"DocumentSourceImportEnvelope",
|
||||
),
|
||||
},
|
||||
},
|
||||
},
|
||||
"200": {
|
||||
description:
|
||||
"Already held (`duplicate`) or deleted in HealthLog (`deleted`).",
|
||||
content: {
|
||||
"application/json": {
|
||||
schema: dataEnvelope(
|
||||
importResultSchema,
|
||||
"DocumentSourceImportHeldEnvelope",
|
||||
),
|
||||
},
|
||||
},
|
||||
},
|
||||
"403": refusal,
|
||||
"404": {
|
||||
description: `${unavailable.description} \`documents.sources.linkTargetNotFound\`: the record in \`link\` is not the caller's own live ${DOCUMENT_PICKER_LINK_KINDS.join(" / ")}.`,
|
||||
content: { "application/json": { schema: errorEnvelope } },
|
||||
},
|
||||
"409": {
|
||||
description:
|
||||
"`documents.inbound.sourceAliasLimit`: these bytes are already stored under too many source ids.",
|
||||
content: { "application/json": { schema: errorEnvelope } },
|
||||
},
|
||||
"413": {
|
||||
description:
|
||||
"`documents.inbound.fileTooLarge` (with `maxFileBytes`) or `documents.inbound.quotaExceeded` (with `quotaBytes` and `usedBytes`).",
|
||||
content: { "application/json": { schema: errorEnvelope } },
|
||||
},
|
||||
"415": {
|
||||
description:
|
||||
"`documents.inbound.fileType`: the file is not a type the vault accepts.",
|
||||
content: { "application/json": { schema: errorEnvelope } },
|
||||
},
|
||||
"422": originRefusal,
|
||||
"502": upstream,
|
||||
"401": stdResponses["401"],
|
||||
"429": {
|
||||
description:
|
||||
"`documents.inbound.rateLimited` (the person's hourly upload allowance, with the rate-limit headers) or `documents.inbound.uploadBusy` (too many uploads in progress; `Retry-After: 1`).",
|
||||
content: { "application/json": { schema: errorEnvelope } },
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
};
|
||||
@@ -24,6 +24,7 @@ import {
|
||||
inboundConfirmSchema,
|
||||
inboundFactEditSchema,
|
||||
DOCUMENT_SOURCE_ID_MAX,
|
||||
DOCUMENT_SOURCE_INSTANCE_MAX,
|
||||
DOCUMENT_SOURCE_SYSTEMS,
|
||||
DOCUMENT_SUMMARY_STATES,
|
||||
INBOUND_DOCUMENT_KINDS,
|
||||
@@ -417,12 +418,20 @@ export const inboundDocumentPaths: NonNullable<ZodOpenApiObject["paths"]> = {
|
||||
description:
|
||||
"Import source key, second half. Both halves or neither; half a key is 422.",
|
||||
},
|
||||
{
|
||||
name: "sourceInstance",
|
||||
in: "query",
|
||||
required: false,
|
||||
schema: { type: "string", maxLength: DOCUMENT_SOURCE_INSTANCE_MAX },
|
||||
description:
|
||||
"Optional, with a key. The instance of `sourceSystem` the id belongs to, as its http(s) address; normalised to the origin (`scheme://host[:port]`). The same id in two instances is two documents. Optional: a key stored without an instance (every upload before v1.39.3) matches any instance of its system, and a key sent without one matches any stored instance. If also sent in the form it must agree.",
|
||||
},
|
||||
],
|
||||
tags: ["Documents"],
|
||||
summary: "Store a document (no extraction)",
|
||||
description:
|
||||
'STORE-ONLY upload. Stores the raw document ENCRYPTED at rest with `status: STORED` and runs NO extraction — provider-free, no AI consent / budget / egress. A file is always filable, even with no document-scan provider configured. `multipart/form-data`: a `file` plus optional `title`, `kind`, `documentDate` (YYYY-MM-DD), and repeated `episodeIds` form fields (pre-link to the caller\'s illness/condition episodes). Accepted types (magic-byte sniffed, never the wire Content-Type): PDF/JPEG/PNG/WebP/GIF render inline; Office (docx/xlsx/pptx/doc/xls/ppt), text/CSV/Markdown/RTF, TIFF, HEIC/HEIF, XML/JSON are stored verbatim and served download-only. HEIC is stored as-is but attachment-only — prefer transcoding to JPEG client-side for inline preview parity. Error contract: `413` with `meta.reason = "fileTooLarge"` (+ `maxFileBytes`) or `"quotaExceeded"` (+ `quotaBytes`, `usedBytes`); `415` with `meta.reason = "unsupportedType"`. A same-user duplicate (same bytes) returns 200 + `meta.duplicate: true` with the existing row — not an error. `Idempotency-Key` honoured. Read `GET /api/documents/inbound/usage` for the effective limits before offering an upload. AI extraction is a separate opt-in action — see `POST /api/documents/inbound/{id}/extract`.\n\n' +
|
||||
"**Importing from another system (v1.39.2).** Optional `sourceSystem` (`PAPERLESS`, `PAPRA`, `OTHER`) and `sourceId` (its id there, printable, up to 128 characters; needs `sourceSystem`) key the upload. A re-send with a known key is a duplicate: 200 with the live row (`meta.duplicate: true`), or, when the owner DELETED that document, 200 with `data.deleted: true` and `meta.deleted: true` and nothing stored; this holds after the 30-day purge too. The key may also ride the query string (`?sourceSystem=…&sourceId=…`): a key the vault already holds is then answered at once, the file is not stored or processed, and the request counts only toward the lookup allowance shared with `GET /api/documents/inbound/source` (5000 an hour per token, per person for a session; 429 past it), not toward the hourly upload limit. A key in both places must agree (else 422). Every other upload counts toward the hourly upload limit, duplicates included. A key that is answered with an existing document by its bytes is remembered for that document, so deleting it keeps that key deleted too; one document takes at most 20 such keys, beyond which the upload is refused 409 (`documents.inbound.sourceAliasLimit`). `OTHER` is one shared namespace per account: two different systems both sent as `OTHER` must not reuse each other\'s ids. `aiRead=defer` holds back automatic AI reading for this upload: the thumbnail and a local text index still run, the summary and lab staging do not, and the document stays out of every automatic AI path (the summary catch-up, the automatic index, Index all for search) until the person reads it with AI or generates its summary. Absent, behaviour is unchanged.\n\n" +
|
||||
"**Importing from another system (v1.39.2).** Optional `sourceSystem` (`PAPERLESS`, `PAPRA`, `OTHER`) and `sourceId` (its id there, printable, up to 128 characters, never `.` or `..`; needs `sourceSystem`) key the upload, optionally with `sourceInstance` (v1.39.3: the address of the Paperless-ngx or Papra it came from, normalised to its origin; without it the key matches any instance of the system). A re-send with a known key is a duplicate: 200 with the live row (`meta.duplicate: true`), or, when the owner DELETED that document, 200 with `data.deleted: true` and `meta.deleted: true` and nothing stored; this holds after the 30-day purge too. The key may also ride the query string (`?sourceSystem=…&sourceId=…`): a key the vault already holds is then answered at once, the file is not stored or processed, and the request counts only toward the lookup allowance shared with `GET /api/documents/inbound/source` (5000 an hour per token, per person for a session; 429 past it), not toward the hourly upload limit. A key in both places must agree (else 422). Every other upload counts toward the hourly upload limit, duplicates included. A key that is answered with an existing document by its bytes is remembered for that document, so deleting it keeps that key deleted too; one document takes at most 20 such keys, beyond which the upload is refused 409 (`documents.inbound.sourceAliasLimit`). `OTHER` is one shared namespace per account: two different systems both sent as `OTHER` must not reuse each other\'s ids. `aiRead=defer` holds back automatic AI reading for this upload: the thumbnail and a local text index still run, the summary and lab staging do not, and the document stays out of every automatic AI path (the summary catch-up, the automatic index, Index all for search) until the person reads it with AI or generates its summary. Absent, behaviour is unchanged.\n\n" +
|
||||
"**Narrow token.** Also reachable with a `documents:write` Bearer (minted at `POST /api/tokens/documents`), which reaches only this route and the source-key lookup `GET /api/documents/inbound/source`. Such a caller draws on a bucket of its own, keyed on the token (default 120 an hour, `DOCUMENT_UPLOAD_LIMIT_PER_HOUR`, clamped 1-1000; the 429 carries `Retry-After` and `X-RateLimit-*`), and gets a `DocumentUploadReceipt` instead of the stored row; its 413 `quotaExceeded` carries no `quotaBytes` / `usedBytes`. The response is an untagged union (row or receipt) on purpose: a discriminator would need a new field on the row every existing client decodes. A cookie or wildcard caller keeps the 60-an-hour per-user bucket and the full row. With the vault module off the answer is 403 `module.disabled` for every caller.",
|
||||
requestBody: {
|
||||
required: true,
|
||||
@@ -461,6 +470,14 @@ export const inboundDocumentPaths: NonNullable<ZodOpenApiObject["paths"]> = {
|
||||
description:
|
||||
"Optional. The document's id in `sourceSystem`; requires it. Unique per account across live and deleted documents.",
|
||||
}),
|
||||
sourceInstance: z
|
||||
.string()
|
||||
.max(DOCUMENT_SOURCE_INSTANCE_MAX)
|
||||
.optional()
|
||||
.meta({
|
||||
description:
|
||||
"Optional; requires `sourceId`. The instance of `sourceSystem` the id belongs to, as its http(s) address; normalised to the origin (`scheme://host[:port]`). The same id in two instances is two documents. Optional: a key stored without an instance (every upload before v1.39.3) matches any instance of its system, and a key sent without one matches any stored instance.",
|
||||
}),
|
||||
aiRead: z.enum(["defer"]).optional().meta({
|
||||
description:
|
||||
"Optional. `defer` holds back automatic AI reading for this upload (local index and thumbnail only).",
|
||||
@@ -527,6 +544,14 @@ export const inboundDocumentPaths: NonNullable<ZodOpenApiObject["paths"]> = {
|
||||
required: true,
|
||||
schema: { type: "string", maxLength: DOCUMENT_SOURCE_ID_MAX },
|
||||
},
|
||||
{
|
||||
name: "sourceInstance",
|
||||
in: "query",
|
||||
required: false,
|
||||
schema: { type: "string", maxLength: DOCUMENT_SOURCE_INSTANCE_MAX },
|
||||
description:
|
||||
"The instance of `sourceSystem` the id belongs to, as its http(s) address; normalised to the origin (`scheme://host[:port]`). The same id in two instances is two documents. Optional: a key stored without an instance (every upload before v1.39.3) matches any instance of its system, and a key sent without one matches any stored instance.",
|
||||
},
|
||||
],
|
||||
responses: {
|
||||
"200": {
|
||||
|
||||
@@ -50,6 +50,7 @@ import { cyclePaths } from "./cycle";
|
||||
import { dashboardWidgetPaths } from "./dashboard";
|
||||
import { biomarkerPaths } from "./biomarkers";
|
||||
import { inboundDocumentPaths } from "./documents";
|
||||
import { documentSourcePaths } from "./document-sources";
|
||||
import { devicePaths } from "./devices";
|
||||
import { exportPaths } from "./export";
|
||||
import { familyHistoryPaths } from "./family-history";
|
||||
@@ -170,6 +171,10 @@ export const openApiPaths: NonNullable<ZodOpenApiObject["paths"]> = {
|
||||
// discover: a copy this instance can no longer decrypt (appended, spread
|
||||
// order is load-bearing).
|
||||
...adminBackupPaths,
|
||||
// v1.39.3 — the document picker: a person's connection to Paperless-ngx or
|
||||
// Papra, searching it and importing picked documents; cookie-only
|
||||
// (appended, spread order is load-bearing).
|
||||
...documentSourcePaths,
|
||||
// The environmental-context overview. Its own module because nothing else
|
||||
// owns the surface (appended, spread order is load-bearing).
|
||||
//
|
||||
|
||||
@@ -81,4 +81,27 @@ export const documentKeys = {
|
||||
* toggle; a flip invalidates the document AI capability probe.
|
||||
*/
|
||||
documentsAutoAiRead: () => ["documents", "auto-ai-read"] as const,
|
||||
/**
|
||||
* The document picker (#1038): its own top-level prefix, deliberately NOT
|
||||
* under `["documents"]`. Every vault write invalidates that prefix, and a
|
||||
* search or tag read here is a request to the person's Paperless-ngx or
|
||||
* Papra instance; a filing action in the vault must not re-send them.
|
||||
*/
|
||||
documentSources: () => ["documentSources"] as const,
|
||||
/** Availability + the person's connections (`GET /api/documents/sources`). */
|
||||
documentSourcesStatus: () => ["documentSources", "status"] as const,
|
||||
/** One source's tags for the filter. */
|
||||
documentSourceTags: (system: string) =>
|
||||
["documentSources", "tags", system] as const,
|
||||
/** One page of one source's search. */
|
||||
documentSourceSearch: (system: string, search: DocumentSourceSearchKey) =>
|
||||
["documentSources", "search", system, search] as const,
|
||||
};
|
||||
|
||||
/** The facets that make one picker search distinct from another. */
|
||||
export interface DocumentSourceSearchKey {
|
||||
q: string;
|
||||
tag?: string;
|
||||
from?: string;
|
||||
to?: string;
|
||||
}
|
||||
@@ -83,6 +83,7 @@ import {
|
||||
} from "@/lib/validations/emergency-profile";
|
||||
import {
|
||||
DOCUMENT_SOURCE_ID_MAX,
|
||||
DOCUMENT_SOURCE_INSTANCE_MAX,
|
||||
DOCUMENT_SOURCE_SYSTEMS,
|
||||
} from "@/lib/validations/inbound-documents";
|
||||
import { REMINDER_EVENT_SOURCES } from "@/lib/measurement-reminders/satisfy";
|
||||
@@ -1636,6 +1637,14 @@ const documentBackupSchema = z
|
||||
// restore cannot write a system the detail sheet has no name for.
|
||||
sourceSystem: z.enum(DOCUMENT_SOURCE_SYSTEMS).nullable().optional(),
|
||||
sourceId: z.string().max(DOCUMENT_SOURCE_ID_MAX).nullable().optional(),
|
||||
// v1.39.3 — the source instance (an origin). Length-bounded here and
|
||||
// normalised on restore; a value that is not an origin restores as null
|
||||
// (matches any instance) rather than failing the restore.
|
||||
sourceInstance: z
|
||||
.string()
|
||||
.max(DOCUMENT_SOURCE_INSTANCE_MAX)
|
||||
.nullable()
|
||||
.optional(),
|
||||
aiReadDeferred: z.boolean().optional(),
|
||||
summary: z.string().nullable().optional(),
|
||||
createdAt: isoDateTime.optional(),
|
||||
|
||||
@@ -0,0 +1,90 @@
|
||||
/**
|
||||
* Request schemas of the document picker (#1038): saving a connection,
|
||||
* searching a source, importing one picked document. No `userId` field in any
|
||||
* of them; the owner is always the authenticated session.
|
||||
*/
|
||||
import { z } from "zod/v4";
|
||||
|
||||
import { DOCUMENT_SOURCE_BASE_URL_MAX } from "@/lib/documents/sources/origins";
|
||||
import {
|
||||
DOCUMENT_PICKER_LINK_KINDS,
|
||||
DOCUMENT_PICKER_MAX_PAGE,
|
||||
DOCUMENT_PICKER_QUERY_MAX,
|
||||
PAPRA_ORG_ID,
|
||||
} from "@/lib/documents/sources/types";
|
||||
import {
|
||||
documentSourceIdSchema,
|
||||
INBOUND_DOCUMENT_KINDS,
|
||||
} from "@/lib/validations/inbound-documents";
|
||||
|
||||
/** Printable ASCII without spaces: an id or a key, never prose. */
|
||||
const printable = /^[\x21-\x7e]+$/u;
|
||||
|
||||
const isoDate = z
|
||||
.string()
|
||||
.regex(/^\d{4}-\d{2}-\d{2}$/u, "Expected a YYYY-MM-DD date");
|
||||
|
||||
/** Longest API token or key a connection stores. */
|
||||
export const DOCUMENT_SOURCE_TOKEN_MAX = 512;
|
||||
|
||||
/**
|
||||
* Save a connection. `token` may be left out when a connection for the system
|
||||
* already exists: the stored one is kept, so the address or the organization
|
||||
* can change without typing the key again. The first save must carry it (the
|
||||
* route says so). `organizationId` is Papra's and required there.
|
||||
*/
|
||||
export const documentSourceSaveSchema = z.object({
|
||||
baseUrl: z.string().trim().min(1).max(DOCUMENT_SOURCE_BASE_URL_MAX),
|
||||
organizationId: z
|
||||
.string()
|
||||
.trim()
|
||||
.regex(PAPRA_ORG_ID, "Expected a Papra organization id (org_…)")
|
||||
.optional(),
|
||||
token: z
|
||||
.string()
|
||||
.trim()
|
||||
.min(1)
|
||||
.max(DOCUMENT_SOURCE_TOKEN_MAX)
|
||||
.regex(printable, "Expected printable characters without spaces")
|
||||
.optional(),
|
||||
});
|
||||
export type DocumentSourceSaveInput = z.infer<typeof documentSourceSaveSchema>;
|
||||
|
||||
export const documentSourceSearchSchema = z
|
||||
.object({
|
||||
q: z.string().trim().max(DOCUMENT_PICKER_QUERY_MAX).default(""),
|
||||
tag: z
|
||||
.string()
|
||||
.trim()
|
||||
.min(1)
|
||||
.max(64)
|
||||
.regex(printable, "Expected printable characters without spaces")
|
||||
.optional(),
|
||||
from: isoDate.optional(),
|
||||
to: isoDate.optional(),
|
||||
page: z.coerce
|
||||
.number()
|
||||
.int()
|
||||
.min(1)
|
||||
.max(DOCUMENT_PICKER_MAX_PAGE)
|
||||
.default(1),
|
||||
})
|
||||
.refine((v) => !v.from || !v.to || v.from <= v.to, {
|
||||
path: ["to"],
|
||||
message: "The end date is before the start date",
|
||||
});
|
||||
|
||||
export const documentSourceImportSchema = z.object({
|
||||
sourceId: documentSourceIdSchema,
|
||||
kind: z.enum(INBOUND_DOCUMENT_KINDS).optional(),
|
||||
/** The record the picker was opened from, to link the document to. */
|
||||
link: z
|
||||
.object({
|
||||
kind: z.enum(DOCUMENT_PICKER_LINK_KINDS),
|
||||
id: z.string().trim().min(1).max(64),
|
||||
})
|
||||
.optional(),
|
||||
});
|
||||
export type DocumentSourceImportInput = z.infer<
|
||||
typeof documentSourceImportSchema
|
||||
>;
|
||||
@@ -701,12 +701,46 @@ export const DOCUMENT_SOURCE_ID_MAX = 128;
|
||||
* The document's id in its source system. Printable ASCII only: it is a key,
|
||||
* not prose, and it is echoed on the detail sheet.
|
||||
*/
|
||||
const documentSourceIdSchema = z
|
||||
export const documentSourceIdSchema = z
|
||||
.string()
|
||||
.trim()
|
||||
.min(1)
|
||||
.max(DOCUMENT_SOURCE_ID_MAX)
|
||||
.regex(/^[\x21-\x7e]+$/u, "Expected printable characters without spaces");
|
||||
.regex(/^[\x21-\x7e]+$/u, "Expected printable characters without spaces")
|
||||
// A path segment of its own name: an importer that builds a URL from the id
|
||||
// must never be handed one that walks up the source's path.
|
||||
.refine((v) => v !== "." && v !== "..", "Not a document id");
|
||||
|
||||
/** Longest source instance (an origin) a key may carry. */
|
||||
export const DOCUMENT_SOURCE_INSTANCE_MAX = 512;
|
||||
|
||||
/**
|
||||
* v1.39.3 — the instance of the source system a key belongs to, as the
|
||||
* normalised origin (`scheme://host[:port]`) of the Paperless-ngx or Papra it
|
||||
* came from, or null when the value is not one. A path, query or credentials
|
||||
* in the value are dropped: two addresses on one origin are one instance.
|
||||
*/
|
||||
export function normaliseSourceInstance(value: string): string | null {
|
||||
try {
|
||||
const url = new URL(value.trim());
|
||||
if (url.protocol !== "http:" && url.protocol !== "https:") return null;
|
||||
if (url.username || url.password) return null;
|
||||
return url.origin;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/** The instance as a request field: validated, then normalised to its origin. */
|
||||
export const documentSourceInstanceSchema = z
|
||||
.string()
|
||||
.trim()
|
||||
.min(1)
|
||||
.max(DOCUMENT_SOURCE_INSTANCE_MAX)
|
||||
.refine((v) => normaliseSourceInstance(v) !== null, {
|
||||
message: "Expected the http(s) address of the source instance",
|
||||
})
|
||||
.transform((v) => normaliseSourceInstance(v) as string);
|
||||
|
||||
/**
|
||||
* A source key on its own — the query-string form the upload accepts ahead of
|
||||
@@ -715,6 +749,8 @@ const documentSourceIdSchema = z
|
||||
export const documentSourceKeySchema = z.object({
|
||||
sourceSystem: z.enum(DOCUMENT_SOURCE_SYSTEMS),
|
||||
sourceId: documentSourceIdSchema,
|
||||
/** Optional: without it the key matches any instance of the system. */
|
||||
sourceInstance: documentSourceInstanceSchema.optional(),
|
||||
});
|
||||
|
||||
export const documentCreateSchema = z
|
||||
@@ -731,6 +767,11 @@ export const documentCreateSchema = z
|
||||
* not prose, and it is echoed on the detail sheet.
|
||||
*/
|
||||
sourceId: documentSourceIdSchema.optional(),
|
||||
/**
|
||||
* v1.39.3 — which instance of `sourceSystem` (its origin). Optional; a
|
||||
* key without one matches any instance.
|
||||
*/
|
||||
sourceInstance: documentSourceInstanceSchema.optional(),
|
||||
/**
|
||||
* `defer` holds back automatic AI reading for this upload: the thumbnail
|
||||
* and a local text index still run, the summary and the lab staging do
|
||||
@@ -741,6 +782,10 @@ export const documentCreateSchema = z
|
||||
.refine((v) => v.sourceId === undefined || v.sourceSystem !== undefined, {
|
||||
path: ["sourceSystem"],
|
||||
message: "sourceId needs a sourceSystem",
|
||||
})
|
||||
.refine((v) => v.sourceInstance === undefined || v.sourceId !== undefined, {
|
||||
path: ["sourceInstance"],
|
||||
message: "sourceInstance needs a sourceId",
|
||||
});
|
||||
|
||||
export type DocumentCreateInput = z.infer<typeof documentCreateSchema>;
|
||||
|
||||
Reference in new issue
Block a user