fix(documents): keep every import key deleted, answer re-sends cheaply

A keyed upload that matched a stored document by its bytes never recorded
its key, so deleting that document let the next import run store it again.
Such keys are now kept as aliases of the document, the source-key check
reads them, and the purge carries them into the ledger with the row's own.

A source key may ride the query string and is then answered before the
body is read or the hourly allowance charged; an upload answered as a
duplicate hands its slot back. GET /api/documents/inbound/source answers
whether a key is held, under the same documents:write scope, so an
importer can skip a download.

aiRead=defer is stored on the row. The summary catch-up, the summary job
and lab auto-staging skip such documents until an explicit Read with AI
or Generate summary clears the marker; backup and restore carry it.

A documents:write caller no longer sees the quota figures on a full vault.

Refs #1038
This commit is contained in:
Marc-André Bombeck committed 2026-09-25 22:35:11 +02:00
1 parent 0d223fdd9c
commit 10a1c3ca2e
30 files changed
+1050 -113

No files matched your search

@@ -15,7 +15,11 @@ import { NextRequest } from "next/server";
vi.mock("@/lib/db", () => ({
prisma: {
inboundDocument: { findFirst: vi.fn(), update: vi.fn() },
inboundDocument: {
findFirst: vi.fn(),
update: vi.fn(),
updateMany: vi.fn().mockResolvedValue({ count: 0 }),
},
extractedFact: { create: vi.fn() },
documentContentIndex: { upsert: vi.fn() },
user: { findUnique: vi.fn() },
@@ -290,6 +294,17 @@ describe("POST /api/documents/inbound/[id]/index — staging continuation", () =
expect(maybeAutoStageLabFacts).toHaveBeenCalledWith("user-1", "doc-1");
const body = await res.json();
expect(body.data.labFactsStaged).toBe(4);
// Read with AI ends an import's hold on automatic AI reading (#1038),
// before the staging that would otherwise refuse it.
expect(prisma.inboundDocument.updateMany).toHaveBeenCalledWith({
where: { id: "doc-1", userId: "user-1", aiReadDeferred: true },
data: { aiReadDeferred: false },
});
expect(
vi.mocked(prisma.inboundDocument.updateMany).mock.invocationCallOrder[0],
).toBeLessThan(
vi.mocked(maybeAutoStageLabFacts).mock.invocationCallOrder[0],
);
});
it("reports zero staged facts as an honest zero, never an error", async () => {
@@ -106,6 +106,13 @@ async function finishIndex(
source,
tokenCount,
});
// Read with AI is the person reading the document on purpose, so an
// import's hold on automatic AI reading (`aiRead=defer`) ends here, before
// the lab staging below that would otherwise refuse it.
await prisma.inboundDocument.updateMany({
where: { id: documentId, userId, aiReadDeferred: true },
data: { aiReadDeferred: false },
});
await auditLog("documents.inbound.index", {
userId,
ipAddress: getClientIp(request),
@@ -192,7 +192,10 @@ describe("POST /api/documents/inbound/[id]/summary", () => {
deletedAt: null,
summaryEncrypted: null,
},
data: expect.objectContaining({ summaryState: "READY" }),
data: expect.objectContaining({
summaryState: "READY",
aiReadDeferred: false,
}),
});
assertNoPersistence();
});
@@ -266,7 +269,7 @@ describe("POST /api/documents/inbound/[id]/summary", () => {
deletedAt: null,
summaryState: { not: "READY" },
},
data: { summaryState: "WITHHELD" },
data: { summaryState: "WITHHELD", aiReadDeferred: false },
});
const calls = vi.mocked(prisma.inboundDocument.updateMany).mock.calls;
for (const [arg] of calls) {
@@ -155,7 +155,9 @@ async function persistSummary(
deletedAt: null,
summaryState: { not: "READY" },
},
data: { summaryState: "WITHHELD" },
// An explicit request is the person reading it on purpose, so an
// import's hold on automatic AI reading ends here.
data: { summaryState: "WITHHELD", aiReadDeferred: false },
});
return "withheld";
}
@@ -173,6 +175,7 @@ async function persistSummary(
summaryEncrypted: encryptDocumentSummary(summary),
summaryGeneratedAt: new Date(),
summaryState: "READY",
aiReadDeferred: false,
},
});
annotate({
@@ -53,6 +53,10 @@ vi.mock("@/lib/db", () => {
documentImportKey: {
findUnique: vi.fn(),
},
documentSourceAlias: {
findUnique: vi.fn(),
createMany: vi.fn(),
},
extractedFact: {
groupBy: vi.fn(),
},
@@ -114,6 +118,7 @@ vi.mock("@/lib/rate-limit", () => ({
resetAt: Date.now() + 3_600_000,
}),
rateLimitHeaders: () => ({}),
refundRateLimit: vi.fn().mockResolvedValue(undefined),
}));
vi.mock("@/lib/auth/session", () => ({ getSession: vi.fn() }));
@@ -146,7 +151,7 @@ import { prisma } from "@/lib/db";
import { getSession } from "@/lib/auth/session";
import { headers } from "next/headers";
import { resolveBearerToken, BearerAuthError } from "@/lib/auth/bearer";
import { checkRateLimit } from "@/lib/rate-limit";
import { checkRateLimit, refundRateLimit } from "@/lib/rate-limit";
import { enqueueDocumentThumbnail } from "@/lib/jobs/document-thumbnail";
import { DOCUMENTS_WRITE_SCOPE } from "@/lib/documents/scopes";
import { requireModuleEnabled } from "@/lib/modules/gate";
@@ -236,6 +241,12 @@ beforeEach(() => {
vi.mocked(prisma.documentImportKey.findUnique).mockResolvedValue(
null as never,
);
vi.mocked(prisma.documentSourceAlias.findUnique).mockResolvedValue(
null as never,
);
vi.mocked(prisma.documentSourceAlias.createMany).mockResolvedValue({
count: 1,
} as never);
// No settings row / no user override → policy defaults (25 MiB / 1 GiB).
vi.mocked(prisma.appSettings.findUnique).mockResolvedValue(null as never);
vi.mocked(prisma.user.findUnique).mockResolvedValue(null as never);
@@ -810,4 +821,129 @@ describe("POST /api/documents/inbound — document import", () => {
expect(enqueueDocumentIndex).toHaveBeenCalledWith("user-1", "doc-1");
expect(enqueueDocumentSummary).toHaveBeenCalledWith("user-1", "doc-1");
});
it("answers a query-string key before charging the bucket or reading the body", async () => {
armToken([DOCUMENTS_WRITE_SCOPE]);
vi.mocked(prisma.inboundDocument.findFirst).mockResolvedValueOnce(
docRow({
id: "doc-src",
sourceSystem: "PAPRA",
sourceId: "doc_1",
}) as never,
);
// Not multipart at all: reading it would be a 400.
const res = await post(
new Request(
"http://localhost/api/documents/inbound?sourceSystem=PAPRA&sourceId=doc_1",
{
method: "POST",
body: "not a form",
headers: { "content-type": "text/plain" },
},
),
);
expect(res.status).toBe(200);
expect((await res.json()).data).toEqual({ id: "doc-src", duplicate: true });
expect(checkRateLimit).not.toHaveBeenCalled();
});
it("422s half a query-string key", async () => {
const res = await post(
new Request("http://localhost/api/documents/inbound?sourceSystem=PAPRA", {
method: "POST",
body: "x",
}),
);
expect(res.status).toBe(422);
expect(checkRateLimit).not.toHaveBeenCalled();
});
it("422s a query key and a form key that disagree", async () => {
const r = mkUpload({ sourceSystem: "PAPRA", sourceId: "other" });
const res = await post(
new Request(`${r.url}?sourceSystem=PAPRA&sourceId=doc_1`, {
method: "POST",
body: await r.arrayBuffer(),
headers: { "content-type": r.headers.get("content-type")! },
}),
);
expect(res.status).toBe(422);
expect(txCreate).not.toHaveBeenCalled();
});
it("remembers the key when keyed bytes are already stored, and refunds the slot", async () => {
// Own-key lookup misses; the content lookup finds a live row.
vi.mocked(prisma.inboundDocument.findFirst)
.mockResolvedValueOnce(null as never)
.mockResolvedValueOnce(docRow({ id: "doc-manual" }) as never);
const res = await post(
mkUpload({ sourceSystem: "PAPRA", sourceId: "doc_9" }),
);
expect(res.status).toBe(200);
expect(prisma.documentSourceAlias.createMany).toHaveBeenCalledWith({
data: [
{
userId: "user-1",
documentId: "doc-manual",
sourceSystem: "PAPRA",
sourceId: "doc_9",
},
],
skipDuplicates: true,
});
expect(refundRateLimit).toHaveBeenCalledWith("documents-upload:user-1");
expect(txCreate).not.toHaveBeenCalled();
});
it("resolves a key held as an alias of a deleted document to deleted", async () => {
vi.mocked(prisma.documentSourceAlias.findUnique).mockResolvedValue({
document: docRow({ id: "doc-gone", deletedAt: new Date() }),
} as never);
const res = await post(
mkUpload({ sourceSystem: "PAPRA", sourceId: "doc_9" }),
);
expect(res.status).toBe(200);
expect((await res.json()).data).toEqual({
id: "doc-gone",
duplicate: true,
deleted: true,
});
expect(refundRateLimit).toHaveBeenCalled();
expect(txCreate).not.toHaveBeenCalled();
});
it("refunds the slot on a plain content duplicate", async () => {
vi.mocked(prisma.inboundDocument.findFirst).mockResolvedValue(
docRow({ id: "doc-existing" }) as never,
);
await post(mkUpload());
expect(refundRateLimit).toHaveBeenCalledWith("documents-upload:user-1");
expect(prisma.documentSourceAlias.createMany).not.toHaveBeenCalled();
});
it("does not refund a stored upload", async () => {
await post(mkUpload());
expect(refundRateLimit).not.toHaveBeenCalled();
});
it("persists the deferral on the row", async () => {
await post(mkUpload({ aiRead: "defer" }));
expect(txCreate.mock.calls[0]![0]!.data.aiReadDeferred).toBe(true);
await post(mkUpload());
expect(txCreate.mock.calls[1]![0]!.data.aiReadDeferred).toBe(false);
});
it("gives a scoped caller no quota figures on a full vault", async () => {
armToken([DOCUMENTS_WRITE_SCOPE]);
vi.mocked(prisma.user.findUnique).mockResolvedValue({
documentQuotaBytes: BigInt(100),
} as never);
txQueryRaw.mockResolvedValue([{ used: BigInt(60) }]);
const res = await post(mkUpload());
expect(res.status).toBe(413);
const meta = (await res.json()).meta;
expect(meta.reason).toBe("quotaExceeded");
expect(meta).not.toHaveProperty("usedBytes");
expect(meta).not.toHaveProperty("quotaBytes");
});
});
+126 -55
View File
@@ -69,6 +69,11 @@ import { enqueueDocumentIndex } from "@/lib/jobs/document-index";
import { enqueueDocumentSummary } from "@/lib/jobs/document-summary";
import { enqueueDocumentThumbnail } from "@/lib/jobs/document-thumbnail";
import { DOCUMENTS_WRITE_SCOPE } from "@/lib/documents/scopes";
import {
findSourceKey,
rememberSourceAlias,
type SourceKeyMatch,
} from "@/lib/documents/source-key";
import {
acquireDocumentUploadSlot,
detectDocumentType,
@@ -81,10 +86,15 @@ import { linkTargets } from "@/lib/links";
import { annotate } from "@/lib/logging/context";
import { requireModuleEnabled } from "@/lib/modules/gate";
import { isP2002 } from "@/lib/prisma-errors";
import { checkRateLimit, rateLimitHeaders } from "@/lib/rate-limit";
import {
checkRateLimit,
rateLimitHeaders,
refundRateLimit,
} from "@/lib/rate-limit";
import {
documentCreateSchema,
documentListQuerySchema,
documentSourceKeySchema,
toContentIndexSource,
type DocumentSourceSystemValue,
} from "@/lib/validations/inbound-documents";
@@ -193,6 +203,16 @@ function deletedResponse(ctx: UploadContext, id: string | null): NextResponse {
return receiptResponse(id, 200, { duplicate: true, deleted: true });
}
/** Answer a source key that is already held: duplicate, or deleted. */
function answerSourceKey(
ctx: UploadContext,
match: SourceKeyMatch,
): Promise<NextResponse> | NextResponse {
return match.state === "live"
? duplicateResponse(ctx, match.document)
: deletedResponse(ctx, match.id);
}
/**
* §3.2 — a duplicate upload is NOT an error: return the existing live row
* with `meta.duplicate: true` at the envelope level (the UI toasts "already
@@ -262,21 +282,66 @@ async function processUpload(
const gate = await requireModuleEnabled(user.id, "inboundDocuments");
if (!gate.enabled) return gate.response;
// A source key may ride the query string. It is answered here, before the
// bucket is charged and before a byte of the body is read: a nightly
// re-sync of an archive HealthLog already holds costs neither. Both halves
// or neither; half a key is a mistake worth saying so about.
const url = new URL(request.url);
const querySystem = url.searchParams.get("sourceSystem");
const queryId = url.searchParams.get("sourceId");
let queryKey: {
sourceSystem: DocumentSourceSystemValue;
sourceId: string;
} | null = null;
if (querySystem !== null || queryId !== null) {
const parsedKey = documentSourceKeySchema.safeParse({
sourceSystem: querySystem ?? undefined,
sourceId: queryId ?? undefined,
});
if (!parsedKey.success) {
return apiValidationError(
"Invalid source key",
sanitiseZodIssues(parsedKey.error.issues),
422,
{ errorCode: "documents.inbound.invalidMetadata" },
);
}
queryKey = parsedKey.data;
const match = await findSourceKey(
user.id,
queryKey.sourceSystem,
queryKey.sourceId,
);
if (match) {
return answerSourceKey(
{
userId: user.id,
scoped,
sourceSystem: queryKey.sourceSystem,
aiDeferred: false,
},
match,
);
}
}
// A scoped token draws on its own bucket, keyed on the token (its id rides
// `session.id` on the Bearer path), so an import running overnight cannot
// use up the allowance the person's own uploads from the web or the phone
// draw on. The cookie / wildcard bucket is unchanged.
const rl = scoped
? await checkRateLimit(
`documents-upload:token:${auth.session.id}`,
resolveDocumentUploadLimitPerHour(),
UPLOAD_WINDOW_MS,
)
: await checkRateLimit(
`documents-upload:${user.id}`,
UPLOAD_LIMIT_PER_HOUR,
UPLOAD_WINDOW_MS,
);
const bucketKey = scoped
? `documents-upload:token:${auth.session.id}`
: `documents-upload:${user.id}`;
const rl = await checkRateLimit(
bucketKey,
scoped ? resolveDocumentUploadLimitPerHour() : UPLOAD_LIMIT_PER_HOUR,
UPLOAD_WINDOW_MS,
);
// A slot pays for a document stored. An upload that turns out to be one
// already held (same key, same bytes) hands its slot back, so re-sends —
// a Paperless workflow firing on every edit, a re-run over an archive —
// never eat into the allowance.
const refundSlot = () => refundRateLimit(bucketKey).catch(() => {});
if (!rl.allowed) {
const response = apiError("Too many uploads. Try again later.", 429, {
errorCode: "documents.inbound.rateLimited",
@@ -356,8 +421,23 @@ async function processUpload(
);
}
const sourceSystem = parsed.data.sourceSystem ?? null;
const sourceId = sourceSystem ? (parsed.data.sourceId ?? null) : null;
// The key from the query string, or from the form fields; if a caller sends
// both they have to agree.
const formSystem = parsed.data.sourceSystem ?? null;
const formId = formSystem ? (parsed.data.sourceId ?? null) : null;
if (
queryKey &&
formSystem !== null &&
(formSystem !== queryKey.sourceSystem || formId !== queryKey.sourceId)
) {
return apiError(
"The source key in the address and in the form differ.",
422,
{ errorCode: "documents.inbound.invalidMetadata" },
);
}
const sourceSystem = queryKey?.sourceSystem ?? formSystem;
const sourceId = queryKey?.sourceId ?? formId;
const ctx: UploadContext = {
userId: user.id,
scoped,
@@ -370,23 +450,12 @@ async function processUpload(
// person deleted is refused a second copy — live and tombstoned rows both
// count (the unique index has no `deleted_at` predicate), and past the purge
// the ledger remembers.
if (sourceSystem && sourceId) {
const keyed = await findBySourceKey(user.id, sourceSystem, sourceId);
if (keyed)
return keyed.deletedAt
? deletedResponse(ctx, keyed.id)
: duplicateResponse(ctx, keyed);
const purged = await prisma.documentImportKey.findUnique({
where: {
userId_sourceSystem_sourceId: {
userId: user.id,
sourceSystem,
sourceId,
},
},
select: { id: true },
});
if (purged) return deletedResponse(ctx, null);
if (sourceSystem && sourceId && !queryKey) {
const match = await findSourceKey(user.id, sourceSystem, sourceId);
if (match) {
await refundSlot();
return answerSourceKey(ctx, match);
}
}
let buffer: Buffer;
@@ -449,6 +518,13 @@ async function processUpload(
omit: { contentEncrypted: true },
});
if (existing) {
// An import sending bytes that are already stored under another key (or
// none) is answered with that document — and the key is remembered for
// it, so once the person deletes the document this key stays deleted too.
if (sourceSystem && sourceId) {
await rememberSourceAlias(user.id, existing.id, sourceSystem, sourceId);
}
await refundSlot();
return duplicateResponse(ctx, existing);
}
@@ -507,6 +583,7 @@ async function processUpload(
: new Date(),
sourceSystem,
sourceId,
aiReadDeferred: ctx.aiDeferred,
},
omit: { contentEncrypted: true },
});
@@ -533,11 +610,14 @@ async function processUpload(
});
} catch (err) {
if (err instanceof QuotaExceededError) {
// The figures are the owner's; a write-only token learns only that the
// vault is full.
return apiError("Storage quota exceeded.", 413, {
errorCode: "documents.inbound.quotaExceeded",
reason: "quotaExceeded",
quotaBytes: limits.quotaBytes,
usedBytes: err.usedBytes,
...(scoped
? {}
: { quotaBytes: limits.quotaBytes, usedBytes: err.usedBytes }),
});
}
if (isP2002(err)) {
@@ -545,18 +625,23 @@ async function processUpload(
// source key, or the same bytes. Surface the winner exactly as the fast
// paths above would have.
if (sourceSystem && sourceId) {
const keyed = await findBySourceKey(user.id, sourceSystem, sourceId);
if (keyed) {
return keyed.deletedAt
? deletedResponse(ctx, keyed.id)
: duplicateResponse(ctx, keyed);
const match = await findSourceKey(user.id, sourceSystem, sourceId);
if (match) {
await refundSlot();
return answerSourceKey(ctx, match);
}
}
const winner = await prisma.inboundDocument.findFirst({
where: { userId: user.id, contentSha256, deletedAt: null },
omit: { contentEncrypted: true },
});
if (winner) return duplicateResponse(ctx, winner);
if (winner) {
if (sourceSystem && sourceId) {
await rememberSourceAlias(user.id, winner.id, sourceSystem, sourceId);
}
await refundSlot();
return duplicateResponse(ctx, winner);
}
}
throw err;
}
@@ -651,24 +736,10 @@ async function processUpload(
);
}
/**
* Find a document by its import source key, live or tombstoned. The blob
* column is never selected.
*/
function findBySourceKey(
userId: string,
sourceSystem: DocumentSourceSystemValue,
sourceId: string,
): Promise<SerialisableDocument | null> {
return prisma.inboundDocument.findFirst({
where: { userId, sourceSystem, sourceId },
omit: { contentEncrypted: true },
});
}
/** Authenticate and reserve memory capacity before any request-body read. */
async function postUpload(request: Request): Promise<Response> {
// The one route that names `documents:write`. Every other vault leg —
// One of the two routes that name `documents:write` (the other is the
// source-key lookup beside it). Every other vault leg —
// list, detail, original, thumbnail, bulk, AI — declares no scope and so
// refuses the token (`bearer-scope-enforcement-guard.test.ts`).
const auth = await requireAuth(DOCUMENTS_WRITE_SCOPE);
@@ -0,0 +1,100 @@
/**
* `GET /api/documents/inbound/source?sourceSystem=…&sourceId=…` — does the
* vault already hold this import source key? (#1038)
*
* Lets an importer skip a document before downloading it from the source
* system at all: a nightly re-sync of an archive HealthLog already holds then
* costs one small request per document instead of a download and an upload.
*
* It answers only what the upload itself would answer for the same key —
* `known`, the document `id`, and whether the owner `deleted` it — so a
* `documents:write` token learns nothing here it could not learn by uploading.
* That token is the reason the route exists, and it is the second of the two
* routes the scope reaches (`bearer-scope-enforcement-guard.test.ts`).
*/
import { apiHandler, isScopedCredential, requireAuth } from "@/lib/api-handler";
import {
apiError,
apiSuccess,
apiValidationError,
sanitiseZodIssues,
} from "@/lib/api-response";
import { DOCUMENTS_WRITE_SCOPE } from "@/lib/documents/scopes";
import { findSourceKey } from "@/lib/documents/source-key";
import { annotate } from "@/lib/logging/context";
import { requireModuleEnabled } from "@/lib/modules/gate";
import { checkRateLimit, rateLimitHeaders } from "@/lib/rate-limit";
import { documentSourceKeySchema } from "@/lib/validations/inbound-documents";
export const dynamic = "force-dynamic";
/**
* Lookups per caller per hour. Generous on purpose: an importer asks once per
* document in the source, so a re-sync of a few thousand documents has to fit
* in an hour. A lookup reads three indexed rows and stores nothing.
*/
const LOOKUP_LIMIT_PER_HOUR = 5000;
const LOOKUP_WINDOW_MS = 60 * 60 * 1000;
export const GET = apiHandler(async (request: Request) => {
const auth = await requireAuth(DOCUMENTS_WRITE_SCOPE);
const { user } = auth;
const gate = await requireModuleEnabled(user.id, "inboundDocuments");
if (!gate.enabled) return gate.response;
const rl = await checkRateLimit(
isScopedCredential(auth)
? `documents-source:token:${auth.session.id}`
: `documents-source:${user.id}`,
LOOKUP_LIMIT_PER_HOUR,
LOOKUP_WINDOW_MS,
);
if (!rl.allowed) {
const response = apiError("Too many lookups. Try again later.", 429, {
errorCode: "documents.inbound.rateLimited",
});
for (const [k, v] of Object.entries(rateLimitHeaders(rl))) {
response.headers.set(k, v);
}
return response;
}
const url = new URL(request.url);
const parsed = documentSourceKeySchema.safeParse({
sourceSystem: url.searchParams.get("sourceSystem") ?? undefined,
sourceId: url.searchParams.get("sourceId") ?? undefined,
});
if (!parsed.success) {
return apiValidationError(
"Invalid source key",
sanitiseZodIssues(parsed.error.issues),
422,
{ errorCode: "documents.inbound.invalidMetadata" },
);
}
const match = await findSourceKey(
user.id,
parsed.data.sourceSystem,
parsed.data.sourceId,
);
annotate({
action: { name: "documents.vault.sourceLookup" },
meta: {
sourceSystem: parsed.data.sourceSystem,
known: match !== null,
deleted: match?.state === "deleted",
},
});
return apiSuccess({
known: match !== null,
id:
match === null
? null
: match.state === "live"
? match.document.id
: match.id,
deleted: match?.state === "deleted",
});
});
+2 -1
View File
@@ -9,7 +9,8 @@
* that file for the reasoning behind each guard. In short:
*
* - `permissions` is a literal, so no request shape reaches it;
* - `documents:write` is accepted by exactly one route, the upload, on the
* - `documents:write` is accepted by the upload and by the source-key lookup
* that answers what the upload would, both on the
* holder's own record, and the answer it gets back is a receipt rather than
* the stored row — it cannot list, read, download, edit or delete a
* document, reach the AI legs, or mint another token;