Merge branch 'feat/document-import-token' into release/v1.39.2

This commit is contained in:
Marc-André Bombeck committed 2026-09-26 00:22:43 +02:00
commit bec79879b5
33 files changed
+1302 -155

No files matched your search

@@ -15,7 +15,11 @@ import { NextRequest } from "next/server";
vi.mock("@/lib/db", () => ({
prisma: {
inboundDocument: { findFirst: vi.fn(), update: vi.fn() },
inboundDocument: {
findFirst: vi.fn(),
update: vi.fn(),
updateMany: vi.fn().mockResolvedValue({ count: 0 }),
},
extractedFact: { create: vi.fn() },
documentContentIndex: { upsert: vi.fn() },
user: { findUnique: vi.fn() },
@@ -290,6 +294,17 @@ describe("POST /api/documents/inbound/[id]/index — staging continuation", () =
expect(maybeAutoStageLabFacts).toHaveBeenCalledWith("user-1", "doc-1");
const body = await res.json();
expect(body.data.labFactsStaged).toBe(4);
// Read with AI ends an import's hold on automatic AI reading (#1038),
// before the staging that would otherwise refuse it.
expect(prisma.inboundDocument.updateMany).toHaveBeenCalledWith({
where: { id: "doc-1", userId: "user-1", aiReadDeferred: true },
data: { aiReadDeferred: false },
});
expect(
vi.mocked(prisma.inboundDocument.updateMany).mock.invocationCallOrder[0],
).toBeLessThan(
vi.mocked(maybeAutoStageLabFacts).mock.invocationCallOrder[0],
);
});
it("reports zero staged facts as an honest zero, never an error", async () => {
@@ -106,6 +106,13 @@ async function finishIndex(
source,
tokenCount,
});
// Read with AI is the person reading the document on purpose, so an
// import's hold on automatic AI reading (`aiRead=defer`) ends here, before
// the lab staging below that would otherwise refuse it.
await prisma.inboundDocument.updateMany({
where: { id: documentId, userId, aiReadDeferred: true },
data: { aiReadDeferred: false },
});
await auditLog("documents.inbound.index", {
userId,
ipAddress: getClientIp(request),
@@ -192,7 +192,10 @@ describe("POST /api/documents/inbound/[id]/summary", () => {
deletedAt: null,
summaryEncrypted: null,
},
data: expect.objectContaining({ summaryState: "READY" }),
data: expect.objectContaining({
summaryState: "READY",
aiReadDeferred: false,
}),
});
assertNoPersistence();
});
@@ -266,7 +269,7 @@ describe("POST /api/documents/inbound/[id]/summary", () => {
deletedAt: null,
summaryState: { not: "READY" },
},
data: { summaryState: "WITHHELD" },
data: { summaryState: "WITHHELD", aiReadDeferred: false },
});
const calls = vi.mocked(prisma.inboundDocument.updateMany).mock.calls;
for (const [arg] of calls) {
@@ -155,7 +155,9 @@ async function persistSummary(
deletedAt: null,
summaryState: { not: "READY" },
},
data: { summaryState: "WITHHELD" },
// An explicit request is the person reading it on purpose, so an
// import's hold on automatic AI reading ends here.
data: { summaryState: "WITHHELD", aiReadDeferred: false },
});
return "withheld";
}
@@ -173,6 +175,7 @@ async function persistSummary(
summaryEncrypted: encryptDocumentSummary(summary),
summaryGeneratedAt: new Date(),
summaryState: "READY",
aiReadDeferred: false,
},
});
annotate({
@@ -53,6 +53,10 @@ vi.mock("@/lib/db", () => {
documentImportKey: {
findUnique: vi.fn(),
},
documentSourceAlias: {
findUnique: vi.fn(),
createMany: vi.fn(),
},
extractedFact: {
groupBy: vi.fn(),
},
@@ -114,6 +118,7 @@ vi.mock("@/lib/rate-limit", () => ({
resetAt: Date.now() + 3_600_000,
}),
rateLimitHeaders: () => ({}),
refundRateLimit: vi.fn().mockResolvedValue(undefined),
}));
vi.mock("@/lib/auth/session", () => ({ getSession: vi.fn() }));
@@ -146,7 +151,7 @@ import { prisma } from "@/lib/db";
import { getSession } from "@/lib/auth/session";
import { headers } from "next/headers";
import { resolveBearerToken, BearerAuthError } from "@/lib/auth/bearer";
import { checkRateLimit } from "@/lib/rate-limit";
import { checkRateLimit, refundRateLimit } from "@/lib/rate-limit";
import { enqueueDocumentThumbnail } from "@/lib/jobs/document-thumbnail";
import { DOCUMENTS_WRITE_SCOPE } from "@/lib/documents/scopes";
import { requireModuleEnabled } from "@/lib/modules/gate";
@@ -236,6 +241,12 @@ beforeEach(() => {
vi.mocked(prisma.documentImportKey.findUnique).mockResolvedValue(
null as never,
);
vi.mocked(prisma.documentSourceAlias.findUnique).mockResolvedValue(
null as never,
);
vi.mocked(prisma.documentSourceAlias.createMany).mockResolvedValue({
count: 1,
} as never);
// No settings row / no user override → policy defaults (25 MiB / 1 GiB).
vi.mocked(prisma.appSettings.findUnique).mockResolvedValue(null as never);
vi.mocked(prisma.user.findUnique).mockResolvedValue(null as never);
@@ -810,4 +821,129 @@ describe("POST /api/documents/inbound — document import", () => {
expect(enqueueDocumentIndex).toHaveBeenCalledWith("user-1", "doc-1");
expect(enqueueDocumentSummary).toHaveBeenCalledWith("user-1", "doc-1");
});
it("answers a query-string key before charging the bucket or reading the body", async () => {
armToken([DOCUMENTS_WRITE_SCOPE]);
vi.mocked(prisma.inboundDocument.findFirst).mockResolvedValueOnce(
docRow({
id: "doc-src",
sourceSystem: "PAPRA",
sourceId: "doc_1",
}) as never,
);
// Not multipart at all: reading it would be a 400.
const res = await post(
new Request(
"http://localhost/api/documents/inbound?sourceSystem=PAPRA&sourceId=doc_1",
{
method: "POST",
body: "not a form",
headers: { "content-type": "text/plain" },
},
),
);
expect(res.status).toBe(200);
expect((await res.json()).data).toEqual({ id: "doc-src", duplicate: true });
expect(checkRateLimit).not.toHaveBeenCalled();
});
it("422s half a query-string key", async () => {
const res = await post(
new Request("http://localhost/api/documents/inbound?sourceSystem=PAPRA", {
method: "POST",
body: "x",
}),
);
expect(res.status).toBe(422);
expect(checkRateLimit).not.toHaveBeenCalled();
});
it("422s a query key and a form key that disagree", async () => {
const r = mkUpload({ sourceSystem: "PAPRA", sourceId: "other" });
const res = await post(
new Request(`${r.url}?sourceSystem=PAPRA&sourceId=doc_1`, {
method: "POST",
body: await r.arrayBuffer(),
headers: { "content-type": r.headers.get("content-type")! },
}),
);
expect(res.status).toBe(422);
expect(txCreate).not.toHaveBeenCalled();
});
it("remembers the key when keyed bytes are already stored, and refunds the slot", async () => {
// Own-key lookup misses; the content lookup finds a live row.
vi.mocked(prisma.inboundDocument.findFirst)
.mockResolvedValueOnce(null as never)
.mockResolvedValueOnce(docRow({ id: "doc-manual" }) as never);
const res = await post(
mkUpload({ sourceSystem: "PAPRA", sourceId: "doc_9" }),
);
expect(res.status).toBe(200);
expect(prisma.documentSourceAlias.createMany).toHaveBeenCalledWith({
data: [
{
userId: "user-1",
documentId: "doc-manual",
sourceSystem: "PAPRA",
sourceId: "doc_9",
},
],
skipDuplicates: true,
});
expect(refundRateLimit).toHaveBeenCalledWith("documents-upload:user-1");
expect(txCreate).not.toHaveBeenCalled();
});
it("resolves a key held as an alias of a deleted document to deleted", async () => {
vi.mocked(prisma.documentSourceAlias.findUnique).mockResolvedValue({
document: docRow({ id: "doc-gone", deletedAt: new Date() }),
} as never);
const res = await post(
mkUpload({ sourceSystem: "PAPRA", sourceId: "doc_9" }),
);
expect(res.status).toBe(200);
expect((await res.json()).data).toEqual({
id: "doc-gone",
duplicate: true,
deleted: true,
});
expect(refundRateLimit).toHaveBeenCalled();
expect(txCreate).not.toHaveBeenCalled();
});
it("refunds the slot on a plain content duplicate", async () => {
vi.mocked(prisma.inboundDocument.findFirst).mockResolvedValue(
docRow({ id: "doc-existing" }) as never,
);
await post(mkUpload());
expect(refundRateLimit).toHaveBeenCalledWith("documents-upload:user-1");
expect(prisma.documentSourceAlias.createMany).not.toHaveBeenCalled();
});
it("does not refund a stored upload", async () => {
await post(mkUpload());
expect(refundRateLimit).not.toHaveBeenCalled();
});
it("persists the deferral on the row", async () => {
await post(mkUpload({ aiRead: "defer" }));
expect(txCreate.mock.calls[0]![0]!.data.aiReadDeferred).toBe(true);
await post(mkUpload());
expect(txCreate.mock.calls[1]![0]!.data.aiReadDeferred).toBe(false);
});
it("gives a scoped caller no quota figures on a full vault", async () => {
armToken([DOCUMENTS_WRITE_SCOPE]);
vi.mocked(prisma.user.findUnique).mockResolvedValue({
documentQuotaBytes: BigInt(100),
} as never);
txQueryRaw.mockResolvedValue([{ used: BigInt(60) }]);
const res = await post(mkUpload());
expect(res.status).toBe(413);
const meta = (await res.json()).meta;
expect(meta.reason).toBe("quotaExceeded");
expect(meta).not.toHaveProperty("usedBytes");
expect(meta).not.toHaveProperty("quotaBytes");
});
});
+126 -55
View File
@@ -69,6 +69,11 @@ import { enqueueDocumentIndex } from "@/lib/jobs/document-index";
import { enqueueDocumentSummary } from "@/lib/jobs/document-summary";
import { enqueueDocumentThumbnail } from "@/lib/jobs/document-thumbnail";
import { DOCUMENTS_WRITE_SCOPE } from "@/lib/documents/scopes";
import {
findSourceKey,
rememberSourceAlias,
type SourceKeyMatch,
} from "@/lib/documents/source-key";
import {
acquireDocumentUploadSlot,
detectDocumentType,
@@ -81,10 +86,15 @@ import { linkTargets } from "@/lib/links";
import { annotate } from "@/lib/logging/context";
import { requireModuleEnabled } from "@/lib/modules/gate";
import { isP2002 } from "@/lib/prisma-errors";
import { checkRateLimit, rateLimitHeaders } from "@/lib/rate-limit";
import {
checkRateLimit,
rateLimitHeaders,
refundRateLimit,
} from "@/lib/rate-limit";
import {
documentCreateSchema,
documentListQuerySchema,
documentSourceKeySchema,
toContentIndexSource,
type DocumentSourceSystemValue,
} from "@/lib/validations/inbound-documents";
@@ -193,6 +203,16 @@ function deletedResponse(ctx: UploadContext, id: string | null): NextResponse {
return receiptResponse(id, 200, { duplicate: true, deleted: true });
}
/** Answer a source key that is already held: duplicate, or deleted. */
function answerSourceKey(
ctx: UploadContext,
match: SourceKeyMatch,
): Promise<NextResponse> | NextResponse {
return match.state === "live"
? duplicateResponse(ctx, match.document)
: deletedResponse(ctx, match.id);
}
/**
* §3.2 — a duplicate upload is NOT an error: return the existing live row
* with `meta.duplicate: true` at the envelope level (the UI toasts "already
@@ -262,21 +282,66 @@ async function processUpload(
const gate = await requireModuleEnabled(user.id, "inboundDocuments");
if (!gate.enabled) return gate.response;
// A source key may ride the query string. It is answered here, before the
// bucket is charged and before a byte of the body is read: a nightly
// re-sync of an archive HealthLog already holds costs neither. Both halves
// or neither; half a key is a mistake worth saying so about.
const url = new URL(request.url);
const querySystem = url.searchParams.get("sourceSystem");
const queryId = url.searchParams.get("sourceId");
let queryKey: {
sourceSystem: DocumentSourceSystemValue;
sourceId: string;
} | null = null;
if (querySystem !== null || queryId !== null) {
const parsedKey = documentSourceKeySchema.safeParse({
sourceSystem: querySystem ?? undefined,
sourceId: queryId ?? undefined,
});
if (!parsedKey.success) {
return apiValidationError(
"Invalid source key",
sanitiseZodIssues(parsedKey.error.issues),
422,
{ errorCode: "documents.inbound.invalidMetadata" },
);
}
queryKey = parsedKey.data;
const match = await findSourceKey(
user.id,
queryKey.sourceSystem,
queryKey.sourceId,
);
if (match) {
return answerSourceKey(
{
userId: user.id,
scoped,
sourceSystem: queryKey.sourceSystem,
aiDeferred: false,
},
match,
);
}
}
// A scoped token draws on its own bucket, keyed on the token (its id rides
// `session.id` on the Bearer path), so an import running overnight cannot
// use up the allowance the person's own uploads from the web or the phone
// draw on. The cookie / wildcard bucket is unchanged.
const rl = scoped
? await checkRateLimit(
`documents-upload:token:${auth.session.id}`,
resolveDocumentUploadLimitPerHour(),
UPLOAD_WINDOW_MS,
)
: await checkRateLimit(
`documents-upload:${user.id}`,
UPLOAD_LIMIT_PER_HOUR,
UPLOAD_WINDOW_MS,
);
const bucketKey = scoped
? `documents-upload:token:${auth.session.id}`
: `documents-upload:${user.id}`;
const rl = await checkRateLimit(
bucketKey,
scoped ? resolveDocumentUploadLimitPerHour() : UPLOAD_LIMIT_PER_HOUR,
UPLOAD_WINDOW_MS,
);
// A slot pays for a document stored. An upload that turns out to be one
// already held (same key, same bytes) hands its slot back, so re-sends —
// a Paperless workflow firing on every edit, a re-run over an archive —
// never eat into the allowance.
const refundSlot = () => refundRateLimit(bucketKey).catch(() => {});
if (!rl.allowed) {
const response = apiError("Too many uploads. Try again later.", 429, {
errorCode: "documents.inbound.rateLimited",
@@ -356,8 +421,23 @@ async function processUpload(
);
}
const sourceSystem = parsed.data.sourceSystem ?? null;
const sourceId = sourceSystem ? (parsed.data.sourceId ?? null) : null;
// The key from the query string, or from the form fields; if a caller sends
// both they have to agree.
const formSystem = parsed.data.sourceSystem ?? null;
const formId = formSystem ? (parsed.data.sourceId ?? null) : null;
if (
queryKey &&
formSystem !== null &&
(formSystem !== queryKey.sourceSystem || formId !== queryKey.sourceId)
) {
return apiError(
"The source key in the address and in the form differ.",
422,
{ errorCode: "documents.inbound.invalidMetadata" },
);
}
const sourceSystem = queryKey?.sourceSystem ?? formSystem;
const sourceId = queryKey?.sourceId ?? formId;
const ctx: UploadContext = {
userId: user.id,
scoped,
@@ -370,23 +450,12 @@ async function processUpload(
// person deleted is refused a second copy — live and tombstoned rows both
// count (the unique index has no `deleted_at` predicate), and past the purge
// the ledger remembers.
if (sourceSystem && sourceId) {
const keyed = await findBySourceKey(user.id, sourceSystem, sourceId);
if (keyed)
return keyed.deletedAt
? deletedResponse(ctx, keyed.id)
: duplicateResponse(ctx, keyed);
const purged = await prisma.documentImportKey.findUnique({
where: {
userId_sourceSystem_sourceId: {
userId: user.id,
sourceSystem,
sourceId,
},
},
select: { id: true },
});
if (purged) return deletedResponse(ctx, null);
if (sourceSystem && sourceId && !queryKey) {
const match = await findSourceKey(user.id, sourceSystem, sourceId);
if (match) {
await refundSlot();
return answerSourceKey(ctx, match);
}
}
let buffer: Buffer;
@@ -449,6 +518,13 @@ async function processUpload(
omit: { contentEncrypted: true },
});
if (existing) {
// An import sending bytes that are already stored under another key (or
// none) is answered with that document — and the key is remembered for
// it, so once the person deletes the document this key stays deleted too.
if (sourceSystem && sourceId) {
await rememberSourceAlias(user.id, existing.id, sourceSystem, sourceId);
}
await refundSlot();
return duplicateResponse(ctx, existing);
}
@@ -507,6 +583,7 @@ async function processUpload(
: new Date(),
sourceSystem,
sourceId,
aiReadDeferred: ctx.aiDeferred,
},
omit: { contentEncrypted: true },
});
@@ -533,11 +610,14 @@ async function processUpload(
});
} catch (err) {
if (err instanceof QuotaExceededError) {
// The figures are the owner's; a write-only token learns only that the
// vault is full.
return apiError("Storage quota exceeded.", 413, {
errorCode: "documents.inbound.quotaExceeded",
reason: "quotaExceeded",
quotaBytes: limits.quotaBytes,
usedBytes: err.usedBytes,
...(scoped
? {}
: { quotaBytes: limits.quotaBytes, usedBytes: err.usedBytes }),
});
}
if (isP2002(err)) {
@@ -545,18 +625,23 @@ async function processUpload(
// source key, or the same bytes. Surface the winner exactly as the fast
// paths above would have.
if (sourceSystem && sourceId) {
const keyed = await findBySourceKey(user.id, sourceSystem, sourceId);
if (keyed) {
return keyed.deletedAt
? deletedResponse(ctx, keyed.id)
: duplicateResponse(ctx, keyed);
const match = await findSourceKey(user.id, sourceSystem, sourceId);
if (match) {
await refundSlot();
return answerSourceKey(ctx, match);
}
}
const winner = await prisma.inboundDocument.findFirst({
where: { userId: user.id, contentSha256, deletedAt: null },
omit: { contentEncrypted: true },
});
if (winner) return duplicateResponse(ctx, winner);
if (winner) {
if (sourceSystem && sourceId) {
await rememberSourceAlias(user.id, winner.id, sourceSystem, sourceId);
}
await refundSlot();
return duplicateResponse(ctx, winner);
}
}
throw err;
}
@@ -651,24 +736,10 @@ async function processUpload(
);
}
/**
* Find a document by its import source key, live or tombstoned. The blob
* column is never selected.
*/
function findBySourceKey(
userId: string,
sourceSystem: DocumentSourceSystemValue,
sourceId: string,
): Promise<SerialisableDocument | null> {
return prisma.inboundDocument.findFirst({
where: { userId, sourceSystem, sourceId },
omit: { contentEncrypted: true },
});
}
/** Authenticate and reserve memory capacity before any request-body read. */
async function postUpload(request: Request): Promise<Response> {
// The one route that names `documents:write`. Every other vault leg —
// One of the two routes that name `documents:write` (the other is the
// source-key lookup beside it). Every other vault leg —
// list, detail, original, thumbnail, bulk, AI — declares no scope and so
// refuses the token (`bearer-scope-enforcement-guard.test.ts`).
const auth = await requireAuth(DOCUMENTS_WRITE_SCOPE);
@@ -0,0 +1,100 @@
/**
* `GET /api/documents/inbound/source?sourceSystem=…&sourceId=…` — does the
* vault already hold this import source key? (#1038)
*
* Lets an importer skip a document before downloading it from the source
* system at all: a nightly re-sync of an archive HealthLog already holds then
* costs one small request per document instead of a download and an upload.
*
* It answers only what the upload itself would answer for the same key —
* `known`, the document `id`, and whether the owner `deleted` it — so a
* `documents:write` token learns nothing here it could not learn by uploading.
* That token is the reason the route exists, and it is the second of the two
* routes the scope reaches (`bearer-scope-enforcement-guard.test.ts`).
*/
import { apiHandler, isScopedCredential, requireAuth } from "@/lib/api-handler";
import {
apiError,
apiSuccess,
apiValidationError,
sanitiseZodIssues,
} from "@/lib/api-response";
import { DOCUMENTS_WRITE_SCOPE } from "@/lib/documents/scopes";
import { findSourceKey } from "@/lib/documents/source-key";
import { annotate } from "@/lib/logging/context";
import { requireModuleEnabled } from "@/lib/modules/gate";
import { checkRateLimit, rateLimitHeaders } from "@/lib/rate-limit";
import { documentSourceKeySchema } from "@/lib/validations/inbound-documents";
export const dynamic = "force-dynamic";
/**
* Lookups per caller per hour. Generous on purpose: an importer asks once per
* document in the source, so a re-sync of a few thousand documents has to fit
* in an hour. A lookup reads three indexed rows and stores nothing.
*/
const LOOKUP_LIMIT_PER_HOUR = 5000;
const LOOKUP_WINDOW_MS = 60 * 60 * 1000;
export const GET = apiHandler(async (request: Request) => {
const auth = await requireAuth(DOCUMENTS_WRITE_SCOPE);
const { user } = auth;
const gate = await requireModuleEnabled(user.id, "inboundDocuments");
if (!gate.enabled) return gate.response;
const rl = await checkRateLimit(
isScopedCredential(auth)
? `documents-source:token:${auth.session.id}`
: `documents-source:${user.id}`,
LOOKUP_LIMIT_PER_HOUR,
LOOKUP_WINDOW_MS,
);
if (!rl.allowed) {
const response = apiError("Too many lookups. Try again later.", 429, {
errorCode: "documents.inbound.rateLimited",
});
for (const [k, v] of Object.entries(rateLimitHeaders(rl))) {
response.headers.set(k, v);
}
return response;
}
const url = new URL(request.url);
const parsed = documentSourceKeySchema.safeParse({
sourceSystem: url.searchParams.get("sourceSystem") ?? undefined,
sourceId: url.searchParams.get("sourceId") ?? undefined,
});
if (!parsed.success) {
return apiValidationError(
"Invalid source key",
sanitiseZodIssues(parsed.error.issues),
422,
{ errorCode: "documents.inbound.invalidMetadata" },
);
}
const match = await findSourceKey(
user.id,
parsed.data.sourceSystem,
parsed.data.sourceId,
);
annotate({
action: { name: "documents.vault.sourceLookup" },
meta: {
sourceSystem: parsed.data.sourceSystem,
known: match !== null,
deleted: match?.state === "deleted",
},
});
return apiSuccess({
known: match !== null,
id:
match === null
? null
: match.state === "live"
? match.document.id
: match.id,
deleted: match?.state === "deleted",
});
});
+2 -1
View File
@@ -9,7 +9,8 @@
* that file for the reasoning behind each guard. In short:
*
* - `permissions` is a literal, so no request shape reaches it;
* - `documents:write` is accepted by exactly one route, the upload, on the
* - `documents:write` is accepted by the upload and by the source-key lookup
* that answers what the upload would, both on the
* holder's own record, and the answer it gets back is a receipt rather than
* the stored row — it cannot list, read, download, edit or delete a
* document, reach the AI legs, or mint another token;