From bb762135dff5711d6b38a7bd858f897b15ef16df Mon Sep 17 00:00:00 2001 From: haritowa <6107211+haritowa@users.noreply.github.com> Date: Wed, 21 Oct 2026 01:35:59 +0000 Subject: [PATCH 01/11] fix(documents): read a PDF's text with the app's own pdfjs Local text extraction went through pdf-parse, which pins its own pdfjs-dist (5.4.296) beside the app's (6.4.299), and both copies take their worker from globalThis.pdfjsWorker. Whichever ran first in a process broke the other: after a text extraction every raster failed with "The API version "6.4.299" does not match the Worker version "5.4.296"" (thumbnails raster-failed, vision reads of a PDF 422 extractFailed), and after a raster the text extraction failed and the local content index stayed empty without a trace. extractPdfText now reads the text layer with getTextContent on the pdfjs-dist the rasterizer uses, so only one pdfjs runs in the server. Pointing pdf-parse at 6.4 instead is not enough: its destroy() calls PDFDocumentProxy.destroy(), which pdfjs 6 no longer has, so every document would skip its teardown. Two tests run text and raster in one process, each order in its own file. --- .../pdfjs-one-version-extract-first.test.ts | 40 ++++++ .../pdfjs-one-version-raster-first.test.ts | 40 ++++++ src/lib/documents/content-index.ts | 2 +- src/lib/documents/index-document.ts | 2 +- src/lib/documents/local-extract.ts | 119 ++++++++++++++---- 5 files changed, 176 insertions(+), 27 deletions(-) create mode 100644 src/lib/documents/__tests__/pdfjs-one-version-extract-first.test.ts create mode 100644 src/lib/documents/__tests__/pdfjs-one-version-raster-first.test.ts diff --git a/src/lib/documents/__tests__/pdfjs-one-version-extract-first.test.ts b/src/lib/documents/__tests__/pdfjs-one-version-extract-first.test.ts new file mode 100644 index 000000000..3b20f39da --- /dev/null +++ b/src/lib/documents/__tests__/pdfjs-one-version-extract-first.test.ts @@ -0,0 +1,40 @@ +import { readFileSync } from "node:fs"; +import { join } from "node:path"; + +import { describe, expect, it, vi } from "vitest"; + +/** + * Text extraction and rasterising share one process, and pdfjs keeps its worker + * on `globalThis.pdfjsWorker`. While `pdf-parse` brought its own pdfjs + * (5.4.296) beside the app's (6.4.299), whichever ran first installed its + * worker and the other refused it: "The API version "6.4.299" does not match + * the Worker version "5.4.296"". Thumbnails failed as `raster-failed`, vision + * reads of a PDF answered 422 and the local text index stayed empty without a + * word. Reading the text through the app's own pdfjs is the fix; this file runs + * extraction first and `pdfjs-one-version-raster-first.test.ts` the reverse, + * each in a fresh process state. + * + * Mutation check: read the text through `pdf-parse` again in `extractPdfText` → + * the raster step here fails. + */ +vi.mock("@/lib/logging/context", () => ({ annotate: vi.fn() })); + +import { extractPdfText } from "../local-extract"; +import { rasterizePdf } from "../rasterize-pdf"; + +const fixture = readFileSync( + join(process.cwd(), "tests", "fixtures", "pdf", "fallback-fonts.pdf"), +); + +describe("one pdfjs for text and raster: text first", () => { + it("extracts, rasterises and extracts again in one process", async () => { + const first = await extractPdfText(fixture); + expect(first.ok).toBe(true); + + const raster = await rasterizePdf(fixture, 1); + expect(raster.ok).toBe(true); + + const again = await extractPdfText(fixture); + expect(again).toEqual(first); + }); +}); diff --git a/src/lib/documents/__tests__/pdfjs-one-version-raster-first.test.ts b/src/lib/documents/__tests__/pdfjs-one-version-raster-first.test.ts new file mode 100644 index 000000000..3986da2fe --- /dev/null +++ b/src/lib/documents/__tests__/pdfjs-one-version-raster-first.test.ts @@ -0,0 +1,40 @@ +import { readFileSync } from "node:fs"; +import { join } from "node:path"; + +import { describe, expect, it, vi } from "vitest"; + +/** + * Text extraction and rasterising share one process, and pdfjs keeps its worker + * on `globalThis.pdfjsWorker`. While `pdf-parse` brought its own pdfjs + * (5.4.296) beside the app's (6.4.299), whichever ran first installed its + * worker and the other refused it: "The API version "6.4.299" does not match + * the Worker version "5.4.296"". Thumbnails failed as `raster-failed`, vision + * reads of a PDF answered 422 and the local text index stayed empty without a + * word. Reading the text through the app's own pdfjs is the fix; the twin runs + * extraction first (`pdfjs-one-version-extract-first.test.ts`) and this file + * the reverse, each in a fresh process state. + * + * Mutation check: read the text through `pdf-parse` again in `extractPdfText` → + * the text step here fails. + */ +vi.mock("@/lib/logging/context", () => ({ annotate: vi.fn() })); + +import { extractPdfText } from "../local-extract"; +import { rasterizePdf } from "../rasterize-pdf"; + +const fixture = readFileSync( + join(process.cwd(), "tests", "fixtures", "pdf", "fallback-fonts.pdf"), +); + +describe("one pdfjs for text and raster: raster first", () => { + it("rasterises, extracts and rasterises again in one process", async () => { + const first = await rasterizePdf(fixture, 1); + expect(first.ok).toBe(true); + + const text = await extractPdfText(fixture); + expect(text.ok).toBe(true); + + const again = await rasterizePdf(fixture, 1); + expect(again.ok).toBe(true); + }); +}); diff --git a/src/lib/documents/content-index.ts b/src/lib/documents/content-index.ts index 4b88616f3..d9f789186 100644 --- a/src/lib/documents/content-index.ts +++ b/src/lib/documents/content-index.ts @@ -325,7 +325,7 @@ export async function loadDocumentChatText( * - `vision` → an AI provider transcribed the stored original (image or PDF, * incl. scanned) — the AI-first primary path. * - `text-ocr` → browser-OCR text posted by the client (opt-in local OCR). - * - `local-pdf` → server-side text-layer extraction of a PDF (`pdf-parse`), no + * - `local-pdf` → server-side text-layer extraction of a PDF (pdfjs), no * provider, no egress — the fallback when no provider is usable. * - `local-ocr` → server-side OCR of a scanned image/PDF (deferred follow-up). */ diff --git a/src/lib/documents/index-document.ts b/src/lib/documents/index-document.ts index 80c3ff25d..aef4348de 100644 --- a/src/lib/documents/index-document.ts +++ b/src/lib/documents/index-document.ts @@ -24,7 +24,7 @@ * configured, the toggle is OFF for an external pick, the provider can't * read this file (e.g. a text-layer PDF on a non-Anthropic account), the * daily budget is reached, or the provider call fails — fall back to - * server-side text-layer extraction (`pdf-parse`, no egress, milliseconds). + * server-side text-layer extraction (pdfjs, no egress, milliseconds). * So content search works at a baseline even with no AI, and every * text-layer PDF is searchable regardless of provider. * diff --git a/src/lib/documents/local-extract.ts b/src/lib/documents/local-extract.ts index d473bd043..8476eb803 100644 --- a/src/lib/documents/local-extract.ts +++ b/src/lib/documents/local-extract.ts @@ -3,9 +3,17 @@ * * When NO AI provider is configured — or the user has not consented to one — * content search still needs a baseline. This module extracts a PDF's embedded - * text layer SERVER-SIDE with `pdf-parse` (already a direct dependency; its - * `getText()` touches only the pure-JS/wasm pdfjs core, never the native - * `@napi-rs/canvas`), in milliseconds, with no OCR and no third-party egress. + * text layer SERVER-SIDE with pdfjs' `getTextContent` (the pure-JS/wasm core, + * never the native `@napi-rs/canvas`), in milliseconds, with no OCR and no + * third-party egress. + * + * It reads through the app's own `pdfjs-dist`, the copy `./rasterize-pdf` + * renders with. It used to go through `pdf-parse`, which brings a second + * pdfjs of its own, and the two cannot share a process: both take their + * worker from `globalThis.pdfjsWorker`, so whichever ran first broke the + * other ("The API version "6.4.299" does not match the Worker version + * "5.4.296""). Thumbnails and vision reads failed after a text extraction, + * and the text extraction failed after a raster. * The extracted text feeds the SAME blind, encrypted content index the provider * path writes (`upsertContentIndex`), so nothing readable is stored at rest. * @@ -20,19 +28,68 @@ */ import { Buffer } from "node:buffer"; -// `pdf-parse` pulls in pdfjs, whose module top-level references the browser-only -// `DOMMatrix` global. A STATIC import makes the Turbopack server chunk evaluate -// that reference at instantiation and throw `DOMMatrix is not defined`, taking -// down every route/worker that shares the chunk (the document routes among -// them). The value is therefore loaded lazily inside `extractPdfText` via a -// runtime `import()`, which lets pdf-parse initialise pdfjs on its own terms and -// never touches DOMMatrix at load — verified to extract a real text layer under -// plain Node. Keep the surface type as a type-only import (fully erased at -// build, no runtime module evaluation). -import type { PDFParse } from "pdf-parse"; - import { annotate } from "@/lib/logging/context"; +// pdfjs' module top-level references the browser-only `DOMMatrix` global. A +// STATIC import makes the Turbopack server chunk evaluate that reference at +// instantiation and throw `DOMMatrix is not defined`, taking down every +// route/worker that shares the chunk (the document routes among them). The +// module is therefore loaded lazily inside `extractPdfText` via a runtime +// `import()`, and only the slice of its API used here is typed, structurally. +interface PdfTextItem { + str: string; + /** [a, b, c, d, x, y]: the run's text matrix; x and y place it on the page. */ + transform: number[]; + width: number; + height: number; + hasEOL: boolean; +} +interface PdfTextPage { + getTextContent(): Promise<{ items: Array }>; + cleanup(): void; +} +interface PdfTextDocument { + numPages: number; + getPage(pageNumber: number): Promise; +} +interface PdfTextLoadingTask { + promise: Promise; + destroy(): Promise; +} +interface PdfTextModule { + getDocument(opts: { + data: Uint8Array; + verbosity?: number; + isEvalSupported?: boolean; + useSystemFonts?: boolean; + }): PdfTextLoadingTask; +} + +/** + * One page's text runs as text: a new line where a run ends its line or the + * next one sits on another, a space where two runs on one line leave a gap, + * nothing where they touch (a word split across two fonts stays one word). + */ +function pageText(items: Array): string { + let text = ""; + let last: { end: number; y: number; size: number } | null = null; + for (const item of items) { + // Marked-content boundaries carry no text. + if (!("str" in item)) continue; + const x = item.transform[4] ?? 0; + const y = item.transform[5] ?? 0; + const size = Math.abs(item.transform[3] ?? 0) || item.height || 1; + if (last && !text.endsWith("\n")) { + if (Math.abs(y - last.y) > last.size / 2) text += "\n"; + else if (x - last.end > size * 0.1 && !/\s$/.test(text)) text += " "; + } + text += item.str; + if (item.hasEOL) text += "\n"; + last = { end: x + item.width, y, size }; + } + return text; +} + /** * Minimum joined text length for a PDF to count as a real text-layer document. * Below this a PDF is treated as scanned/empty — a few stray glyphs from a form @@ -68,19 +125,31 @@ export type LocalExtractResult = export async function extractPdfText( buffer: Buffer, ): Promise { - let parser: PDFParse | null = null; + let task: PdfTextLoadingTask | null = null; try { // Lazy load: keeps pdfjs/DOMMatrix out of the eager server chunk (see the // import-site note). Resolved at first PDF extraction, then module-cached. - const { PDFParse } = await import("pdf-parse"); - parser = new PDFParse({ data: new Uint8Array(buffer) }); - const result = await parser.getText({ - // Bound the page walk and drop the default "-- N of M --" page markers so - // only real document text is tokenised. - first: LOCAL_MAX_PAGES, - pageJoiner: "", + const pdfjs = + (await import("pdfjs-dist/legacy/build/pdf.mjs")) as unknown as PdfTextModule; + task = pdfjs.getDocument({ + data: new Uint8Array(buffer), + verbosity: 0, // VerbosityLevel.ERRORS — no console spam on odd PDFs. + isEvalSupported: false, // never eval font programs (defence in depth). + useSystemFonts: false, }); - const text = (result.text ?? "").trim(); + const doc = await task.promise; + // Bound the page walk so a pathological many-page PDF costs no more. + const pageCount = Math.min(doc.numPages, LOCAL_MAX_PAGES); + const pages: string[] = []; + for (let n = 1; n <= pageCount; n++) { + const page = await doc.getPage(n); + try { + pages.push(pageText((await page.getTextContent()).items)); + } finally { + page.cleanup(); + } + } + const text = pages.join("\n").trim(); if (text.length < LOCAL_TEXT_MIN_CHARS) { return { ok: false, reason: "empty" }; } @@ -88,9 +157,9 @@ export async function extractPdfText( } catch { return { ok: false, reason: "error" }; } finally { - if (parser) { + if (task) { try { - await parser.destroy(); + await task.destroy(); } catch { // Best-effort teardown; a destroy failure must not mask the result. } From 275121391b3341c0994268e38d840b8355ad4b5a Mon Sep 17 00:00:00 2001 From: haritowa <6107211+haritowa@users.noreply.github.com> Date: Thu, 22 Oct 2026 16:48:16 +0000 Subject: [PATCH 02/11] test(documents): keep pdf-parse out of the server's runtime A structural guard fails when any runtime file under src/ imports pdf-parse again, which would bring its second pdfjs back into the server process. The next.config comment on the tracing includes now describes the text path as it is, and the extractor no longer turns an empty end-of-line run into a blank line of its own. --- next.config.ts | 13 ++-- src/__tests__/single-pdfjs-guard.test.ts | 78 ++++++++++++++++++++++++ src/lib/documents/local-extract.ts | 5 +- 3 files changed, 90 insertions(+), 6 deletions(-) create mode 100644 src/__tests__/single-pdfjs-guard.test.ts diff --git a/next.config.ts b/next.config.ts index 055b1dcf7..66662bb82 100644 --- a/next.config.ts +++ b/next.config.ts @@ -302,11 +302,14 @@ const nextConfig: NextConfig = { outputFileTracingIncludes: { "*": [ "./src/lib/ai/prompts/safety-contracts.*.yaml", - // Document AI local text extraction runs `pdf-parse` (→ pdfjs-dist) - // server-side in the content-index job — the provider-free fallback that - // keeps content search working with no AI configured. Both are pure JS + - // wasm; there is NO native `.node` on the text-layer path (`@napi-rs/canvas` - // is only pulled for image rendering, which `getText()` never invokes). + // Document AI local text extraction reads the text layer with pdfjs-dist + // (`getTextContent`, `src/lib/documents/local-extract.ts`) server-side in + // the content-index job — the provider-free fallback that keeps content + // search working with no AI configured. It is pure JS + wasm; there is NO + // native `.node` on the text-layer path (`@napi-rs/canvas` is only pulled + // for image rendering). Nothing at runtime imports `pdf-parse` any more + // (its own older pdfjs clashed with the app's); its glob below is dead + // weight kept until the image is rebuilt and checked without it. // The standalone tracer can miss pdfjs-dist's wasm decoders (they resolve // via `new URL(..., import.meta.url)`), so pin both packages into the // runtime image explicitly. pnpm keeps pdfjs-dist unhoisted under `.pnpm`. diff --git a/src/__tests__/single-pdfjs-guard.test.ts b/src/__tests__/single-pdfjs-guard.test.ts new file mode 100644 index 000000000..897189e1b --- /dev/null +++ b/src/__tests__/single-pdfjs-guard.test.ts @@ -0,0 +1,78 @@ +/** + * Structural guard: the server runs one pdfjs. + * + * ## The failure this freezes + * + * `pdf-parse` brings its own `pdfjs-dist` beside the app's, and under Node + * every pdfjs copy keeps its worker on the one `globalThis.pdfjsWorker`. While + * local text extraction went through pdf-parse, whichever copy ran first in a + * process broke the other: "The API version "6.4.299" does not match the + * Worker version "5.4.296"". Thumbnails and vision reads failed after a text + * extraction, and the text extraction failed after a raster. + * + * ## What it proves and what it does not + * + * No runtime file under `src/` imports `pdf-parse`, so the only pdfjs the + * server loads is the app's own. Tests may still read generated PDFs back with + * it; they run in their own processes. The pdfjs-one-version tests prove the + * two orders in one process; this file stops the import from coming back. + */ +import { readFileSync } from "node:fs"; +import { join } from "node:path"; + +import { describe, expect, it } from "vitest"; + +import { + requireFloor, + stripComments, + walkSourceFiles, +} from "./helpers/source-files"; + +const SRC = join(process.cwd(), "src"); + +/** + * Every runtime module under `src/`: the suites, mocks and generated client + * are not loaded by the server. About 3,400 today; the floor sits below that + * with headroom, so a walk that comes back short fails instead of passing. + */ +function runtimeFiles(): string[] { + const files = walkSourceFiles(SRC, { + floor: 5000, + extensions: [".ts", ".tsx", ".js", ".jsx", ".mjs", ".cjs", ".mts", ".cts"], + }).filter( + (rel) => + !rel.startsWith("generated/") && + !rel.split("/").includes("__tests__") && + !rel.split("/").includes("__mocks__") && + !/\.test\.(ts|tsx)$/.test(rel), + ); + return [...requireFloor("runtime files under src/", files, 3000)]; +} + +/** A static import, a re-export, a dynamic import or a require of pdf-parse. */ +const PDF_PARSE_IMPORT = + /(?:\bfrom\s*|\bimport\s*\(\s*|\brequire\s*\(\s*|\bimport\s+)["']pdf-parse(?:\/[^"']*)?["']/; + +describe("one pdfjs in the server", () => { + it("no runtime file imports pdf-parse", () => { + const importers = runtimeFiles().filter((rel) => + PDF_PARSE_IMPORT.test( + stripComments(readFileSync(join(SRC, rel), "utf8")), + ), + ); + expect(importers).toEqual([]); + }); + + it("recognises each way of importing it", () => { + for (const line of [ + 'import { PDFParse } from "pdf-parse";', + 'import type { PDFParse } from "pdf-parse";', + 'const { PDFParse } = await import("pdf-parse");', + "const m = require('pdf-parse/worker');", + 'export { PDFParse } from "pdf-parse";', + ]) { + expect(PDF_PARSE_IMPORT.test(line), line).toBe(true); + } + expect(PDF_PARSE_IMPORT.test('import x from "pdf-parser";')).toBe(false); + }); +}); diff --git a/src/lib/documents/local-extract.ts b/src/lib/documents/local-extract.ts index 8476eb803..39b33df09 100644 --- a/src/lib/documents/local-extract.ts +++ b/src/lib/documents/local-extract.ts @@ -84,7 +84,10 @@ function pageText(items: Array): string { else if (x - last.end > size * 0.1 && !/\s$/.test(text)) text += " "; } text += item.str; - if (item.hasEOL) text += "\n"; + // An empty run that only ends a line adds no blank line of its own. + if (item.hasEOL && !(item.str === "" && text.endsWith("\n"))) { + text += "\n"; + } last = { end: x + item.width, y, size }; } return text; From 5090a6fdc1ad41d9aaac5edeb6bcad4da4c52460 Mon Sep 17 00:00:00 2001 From: haritowa <6107211+haritowa@users.noreply.github.com> Date: Fri, 23 Oct 2026 17:31:25 +0000 Subject: [PATCH 03/11] chore(deps): say what the pdf-parse copy of pdfjs is still for The override comment said pdf-parse's pdfjs 5.4.296 drives local text extraction. Extraction now reads with the app's own pdfjs; only the tests still use pdf-parse, which has to stay on pdfjs 5 because its teardown calls an API pdfjs 6 removed. The range itself is unchanged. --- pnpm-workspace.yaml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index 2f95c35f1..aade691c4 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -73,8 +73,10 @@ overrides: # transitively, and `outputFileTracingIncludes` ships EVERY `pdfjs-dist@*` in # the store into the standalone image — so an unused transitive copy is still # a copy in the runtime image. The range is bounded on both ends deliberately: - # `pdf-parse` resolves 5.4.296, which predates the flaw and drives the local - # text-extraction path, so it must not be dragged onto a new major. + # `pdf-parse` resolves 5.4.296, which predates the flaw. Only the tests use it + # now (they read generated PDFs back); it must not be dragged onto a new + # major, because its teardown calls `PDFDocumentProxy.destroy()`, which pdfjs + # 6 no longer has. Local text extraction reads with the app's own pdfjs. "pdfjs-dist@>=5.6.83 <6.2.108": "6.2.108" # Deeply nested input drives DeepmergeTS into unbounded recursion until the # stack gives out. The copy here is Prisma's: `@prisma/config` still asks for From 85bef667dd0d9d5052da948877090fb752f1088d Mon Sep 17 00:00:00 2001 From: haritowa <6107211+haritowa@users.noreply.github.com> Date: Fri, 23 Oct 2026 17:56:38 +0000 Subject: [PATCH 04/11] test(documents): read files the way the realism ratchet expects The test-realism ratchet counts a test that calls readFileSync and names a source file. The guard now reads source through readSourceText, and the two one-process tests no longer name their twin file in a comment. Their doc comments are rewritten in plainer words. --- src/__tests__/single-pdfjs-guard.test.ts | 6 ++--- .../pdfjs-one-version-extract-first.test.ts | 25 +++++++++++-------- .../pdfjs-one-version-raster-first.test.ts | 25 +++++++++++-------- 3 files changed, 30 insertions(+), 26 deletions(-) diff --git a/src/__tests__/single-pdfjs-guard.test.ts b/src/__tests__/single-pdfjs-guard.test.ts index 897189e1b..d42270995 100644 --- a/src/__tests__/single-pdfjs-guard.test.ts +++ b/src/__tests__/single-pdfjs-guard.test.ts @@ -17,12 +17,12 @@ * it; they run in their own processes. The pdfjs-one-version tests prove the * two orders in one process; this file stops the import from coming back. */ -import { readFileSync } from "node:fs"; import { join } from "node:path"; import { describe, expect, it } from "vitest"; import { + readSourceText, requireFloor, stripComments, walkSourceFiles, @@ -56,9 +56,7 @@ const PDF_PARSE_IMPORT = describe("one pdfjs in the server", () => { it("no runtime file imports pdf-parse", () => { const importers = runtimeFiles().filter((rel) => - PDF_PARSE_IMPORT.test( - stripComments(readFileSync(join(SRC, rel), "utf8")), - ), + PDF_PARSE_IMPORT.test(stripComments(readSourceText(join(SRC, rel)))), ); expect(importers).toEqual([]); }); diff --git a/src/lib/documents/__tests__/pdfjs-one-version-extract-first.test.ts b/src/lib/documents/__tests__/pdfjs-one-version-extract-first.test.ts index 3b20f39da..298696510 100644 --- a/src/lib/documents/__tests__/pdfjs-one-version-extract-first.test.ts +++ b/src/lib/documents/__tests__/pdfjs-one-version-extract-first.test.ts @@ -4,18 +4,21 @@ import { join } from "node:path"; import { describe, expect, it, vi } from "vitest"; /** - * Text extraction and rasterising share one process, and pdfjs keeps its worker - * on `globalThis.pdfjsWorker`. While `pdf-parse` brought its own pdfjs - * (5.4.296) beside the app's (6.4.299), whichever ran first installed its - * worker and the other refused it: "The API version "6.4.299" does not match - * the Worker version "5.4.296"". Thumbnails failed as `raster-failed`, vision - * reads of a PDF answered 422 and the local text index stayed empty without a - * word. Reading the text through the app's own pdfjs is the fix; this file runs - * extraction first and `pdfjs-one-version-raster-first.test.ts` the reverse, - * each in a fresh process state. + * Text extraction and rasterising run in one process. In Node, pdfjs keeps its + * worker in one global, `globalThis.pdfjsWorker`. When `pdf-parse` brought its + * own pdfjs (5.4.296) beside the app's (6.4.299), the copy that ran first set + * that global, and the other copy refused it: "The API version "6.4.299" does + * not match the Worker version "5.4.296"". Thumbnails then failed as + * `raster-failed`, vision reads of a PDF answered 422, and the local text index + * stayed empty with no log line. The fix: read the text with the app's own + * pdfjs. * - * Mutation check: read the text through `pdf-parse` again in `extractPdfText` → - * the raster step here fails. + * This file extracts text first and rasterises second. Its twin in this + * folder runs the reverse order. Each order needs its own file, because a + * process keeps the first worker it loads. + * + * Mutation check: read the text through `pdf-parse` again in + * `extractPdfText` → the raster step here fails. */ vi.mock("@/lib/logging/context", () => ({ annotate: vi.fn() })); diff --git a/src/lib/documents/__tests__/pdfjs-one-version-raster-first.test.ts b/src/lib/documents/__tests__/pdfjs-one-version-raster-first.test.ts index 3986da2fe..3389f078e 100644 --- a/src/lib/documents/__tests__/pdfjs-one-version-raster-first.test.ts +++ b/src/lib/documents/__tests__/pdfjs-one-version-raster-first.test.ts @@ -4,18 +4,21 @@ import { join } from "node:path"; import { describe, expect, it, vi } from "vitest"; /** - * Text extraction and rasterising share one process, and pdfjs keeps its worker - * on `globalThis.pdfjsWorker`. While `pdf-parse` brought its own pdfjs - * (5.4.296) beside the app's (6.4.299), whichever ran first installed its - * worker and the other refused it: "The API version "6.4.299" does not match - * the Worker version "5.4.296"". Thumbnails failed as `raster-failed`, vision - * reads of a PDF answered 422 and the local text index stayed empty without a - * word. Reading the text through the app's own pdfjs is the fix; the twin runs - * extraction first (`pdfjs-one-version-extract-first.test.ts`) and this file - * the reverse, each in a fresh process state. + * Text extraction and rasterising run in one process. In Node, pdfjs keeps its + * worker in one global, `globalThis.pdfjsWorker`. When `pdf-parse` brought its + * own pdfjs (5.4.296) beside the app's (6.4.299), the copy that ran first set + * that global, and the other copy refused it: "The API version "6.4.299" does + * not match the Worker version "5.4.296"". Thumbnails then failed as + * `raster-failed`, vision reads of a PDF answered 422, and the local text index + * stayed empty with no log line. The fix: read the text with the app's own + * pdfjs. * - * Mutation check: read the text through `pdf-parse` again in `extractPdfText` → - * the text step here fails. + * This file rasterises first and extracts text second. Its twin in this + * folder runs the reverse order. Each order needs its own file, because a + * process keeps the first worker it loads. + * + * Mutation check: read the text through `pdf-parse` again in + * `extractPdfText` → the text step here fails. */ vi.mock("@/lib/logging/context", () => ({ annotate: vi.fn() })); From 2c5e9a80b9ee6ba43dde7cd4166addc573c0d535 Mon Sep 17 00:00:00 2001 From: haritowa <6107211+haritowa@users.noreply.github.com> Date: Fri, 23 Oct 2026 18:35:17 +0000 Subject: [PATCH 05/11] refactor(documents): type the text read with pdfjs' own declarations extractPdfText cast the dynamic import to a hand-written shape (`as unknown as`), so the compiler could not check the call against pdfjs. It now uses pdfjs' own types: PDFDocumentLoadingTask, TextItem and TextMarkedContent, all type-only imports that are erased at build. That check found a dead option at once: pdfjs 6 has no isEvalSupported. The option went away with the font-program eval path it used to switch off, so it did nothing here; it is removed. --- src/lib/documents/local-extract.ts | 45 ++++++++---------------------- 1 file changed, 11 insertions(+), 34 deletions(-) diff --git a/src/lib/documents/local-extract.ts b/src/lib/documents/local-extract.ts index 39b33df09..98afd9935 100644 --- a/src/lib/documents/local-extract.ts +++ b/src/lib/documents/local-extract.ts @@ -28,6 +28,12 @@ */ import { Buffer } from "node:buffer"; +import type { PDFDocumentLoadingTask } from "pdfjs-dist"; +import type { + TextItem, + TextMarkedContent, +} from "pdfjs-dist/types/src/display/api"; + import { annotate } from "@/lib/logging/context"; // pdfjs' module top-level references the browser-only `DOMMatrix` global. A @@ -35,42 +41,15 @@ import { annotate } from "@/lib/logging/context"; // instantiation and throw `DOMMatrix is not defined`, taking down every // route/worker that shares the chunk (the document routes among them). The // module is therefore loaded lazily inside `extractPdfText` via a runtime -// `import()`, and only the slice of its API used here is typed, structurally. -interface PdfTextItem { - str: string; - /** [a, b, c, d, x, y]: the run's text matrix; x and y place it on the page. */ - transform: number[]; - width: number; - height: number; - hasEOL: boolean; -} -interface PdfTextPage { - getTextContent(): Promise<{ items: Array }>; - cleanup(): void; -} -interface PdfTextDocument { - numPages: number; - getPage(pageNumber: number): Promise; -} -interface PdfTextLoadingTask { - promise: Promise; - destroy(): Promise; -} -interface PdfTextModule { - getDocument(opts: { - data: Uint8Array; - verbosity?: number; - isEvalSupported?: boolean; - useSystemFonts?: boolean; - }): PdfTextLoadingTask; -} +// `import()`. The type imports above are erased at build, so they load +// nothing; they let the compiler check this file against pdfjs' own API. /** * One page's text runs as text: a new line where a run ends its line or the * next one sits on another, a space where two runs on one line leave a gap, * nothing where they touch (a word split across two fonts stays one word). */ -function pageText(items: Array): string { +function pageText(items: Array): string { let text = ""; let last: { end: number; y: number; size: number } | null = null; for (const item of items) { @@ -128,16 +107,14 @@ export type LocalExtractResult = export async function extractPdfText( buffer: Buffer, ): Promise { - let task: PdfTextLoadingTask | null = null; + let task: PDFDocumentLoadingTask | null = null; try { // Lazy load: keeps pdfjs/DOMMatrix out of the eager server chunk (see the // import-site note). Resolved at first PDF extraction, then module-cached. - const pdfjs = - (await import("pdfjs-dist/legacy/build/pdf.mjs")) as unknown as PdfTextModule; + const pdfjs = await import("pdfjs-dist/legacy/build/pdf.mjs"); task = pdfjs.getDocument({ data: new Uint8Array(buffer), verbosity: 0, // VerbosityLevel.ERRORS — no console spam on odd PDFs. - isEvalSupported: false, // never eval font programs (defence in depth). useSystemFonts: false, }); const doc = await task.promise; From ce6c7528640a5bea0d84f1a1d829fa4df8f764ff Mon Sep 17 00:00:00 2001 From: haritowa <6107211+haritowa@users.noreply.github.com> Date: Fri, 23 Oct 2026 18:37:23 +0000 Subject: [PATCH 06/11] refactor(documents): name the steps of the page text joiner pageText worked on raw text-matrix slots and bare numbers. Each step now has a name: positionOf reads where a run starts and ends and its font size, startsNewLine says when the baseline moved by more than half a font size, and leavesWordGap says when the gap is wider than a tenth of the font size. The output is the same on every test PDF. --- src/lib/documents/local-extract.ts | 75 +++++++++++++++++++++++------- 1 file changed, 59 insertions(+), 16 deletions(-) diff --git a/src/lib/documents/local-extract.ts b/src/lib/documents/local-extract.ts index 98afd9935..e726f4a27 100644 --- a/src/lib/documents/local-extract.ts +++ b/src/lib/documents/local-extract.ts @@ -45,29 +45,72 @@ import { annotate } from "@/lib/logging/context"; // nothing; they let the compiler check this file against pdfjs' own API. /** - * One page's text runs as text: a new line where a run ends its line or the - * next one sits on another, a space where two runs on one line leave a gap, - * nothing where they touch (a word split across two fonts stays one word). + * Where a text run sits on the page. pdfjs gives each run a text matrix + * `[a, b, c, d, x, y]`: `x` and `y` are where the run starts, and `d` is its + * vertical scale, which for upright text is the font size. + */ +interface RunPosition { + x: number; + /** The baseline the run sits on. */ + y: number; + /** Where the run ends: its start plus its width. */ + end: number; + fontSize: number; +} + +function positionOf(run: TextItem): RunPosition { + const [, , , scaleY, x = 0, y = 0] = run.transform as number[]; + return { + x, + y, + end: x + run.width, + fontSize: Math.abs(scaleY ?? 0) || run.height || 1, + }; +} + +/** + * The next run is on a new line when its baseline moved by more than half a + * font size. Runs on one line can sit a little higher or lower (a superscript, + * a mixed font), so a small shift does not count. + */ +function startsNewLine(previous: RunPosition, next: RunPosition): boolean { + return Math.abs(next.y - previous.y) > previous.fontSize / 2; +} + +/** + * Two runs on one line are separate words when the gap between them is wider + * than a tenth of the font size. Runs that touch, such as a word drawn in two + * fonts, stay one word. + */ +function leavesWordGap(previous: RunPosition, next: RunPosition): boolean { + return next.x - previous.end > next.fontSize / 10; +} + +/** + * One page's text: its runs in order, with a line break where a run ends a + * line or the next one starts a new line, and a space where two runs on one + * line leave a gap between words. */ function pageText(items: Array): string { let text = ""; - let last: { end: number; y: number; size: number } | null = null; + let previous: RunPosition | null = null; + for (const item of items) { - // Marked-content boundaries carry no text. + // Marked-content markers carry no text. if (!("str" in item)) continue; - const x = item.transform[4] ?? 0; - const y = item.transform[5] ?? 0; - const size = Math.abs(item.transform[3] ?? 0) || item.height || 1; - if (last && !text.endsWith("\n")) { - if (Math.abs(y - last.y) > last.size / 2) text += "\n"; - else if (x - last.end > size * 0.1 && !/\s$/.test(text)) text += " "; + const position = positionOf(item); + + if (previous && !text.endsWith("\n")) { + if (startsNewLine(previous, position)) { + text += "\n"; + } else if (leavesWordGap(previous, position) && !/\s$/.test(text)) { + text += " "; + } } text += item.str; - // An empty run that only ends a line adds no blank line of its own. - if (item.hasEOL && !(item.str === "" && text.endsWith("\n"))) { - text += "\n"; - } - last = { end: x + item.width, y, size }; + if (item.hasEOL && !text.endsWith("\n")) text += "\n"; + + previous = position; } return text; } From 540c6bfaeb3045513a8cf93d5706c96b3f19ec6f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Sat, 24 Oct 2026 20:16:01 +0200 Subject: [PATCH 07/11] refactor(documents): read the text layer through the one shared loader The text read and the rasterizer each opened pdf.js on their own: the text read with bare options, the rasterizer through src/lib/documents/pdfjs.ts with the standard font, CMap and wasm directories. The loader now serves both, so a CID font with a predefined CMap is mapped to characters on the text path too, and there is one set of options to keep in step. The loader is typed with pdf.js' own declarations instead of a hand-written shape. That drops isEvalSupported, which pdf.js 6 no longer has and which did nothing. Only the render slice the DOM-typed declarations cannot describe (the native canvas factory) keeps a named local type in the rasterizer. The run joiner with its named steps moves into the loader as joinTextRuns, and readPdfText is its one page walk; the second layout copy that mimicked the previous library's tabs and blank lines is gone. extractPdfText keeps its outcomes and page cap on top of it, and a failed read still says what pdf.js threw (documents.local_extract.failed). A test draws a word in two fonts into a real PDF and reads it back whole. Refs #1277 --- .../__tests__/pdfjs-text-layer.test.ts | 118 +++++---- src/lib/documents/local-extract.ts | 154 +++--------- src/lib/documents/pdfjs.ts | 232 +++++++----------- src/lib/documents/rasterize-pdf.ts | 54 ++-- 4 files changed, 224 insertions(+), 334 deletions(-) diff --git a/src/lib/documents/__tests__/pdfjs-text-layer.test.ts b/src/lib/documents/__tests__/pdfjs-text-layer.test.ts index aae80e893..ad331d608 100644 --- a/src/lib/documents/__tests__/pdfjs-text-layer.test.ts +++ b/src/lib/documents/__tests__/pdfjs-text-layer.test.ts @@ -1,80 +1,89 @@ import { jsPDF } from "jspdf"; +import type { + TextItem, + TextMarkedContent, +} from "pdfjs-dist/types/src/display/api"; import { describe, expect, it } from "vitest"; -import { pageTextLayer, readPdfText, type PdfTextItem } from "../pdfjs"; +import { joinTextRuns, readPdfText } from "../pdfjs"; /** - * The text layer as the content index reads it. The layout is the one the - * local text read produced before it moved onto the app's own pdf.js (#1277): - * a break where pdf.js marks one or the next item sits on a lower line, a tab - * between far-apart items on one line, each page followed by a blank line. + * The text layer as the content index reads it (#1277): runs in order, a line + * break where a run ends its line or the next one sits on another, a space + * where two runs on one line leave a gap, and nothing where they touch, so a + * word drawn in two fonts stays one word and a search for it still finds it. */ -/** A page whose viewport maps PDF space to itself, flipped like pdf.js. */ -function pageOf(items: PdfTextItem[]) { - return { - getViewport: () => ({ - width: 600, - height: 800, - convertToViewportPoint: (x: number, y: number) => [x, 800 - y], - }), - getTextContent: async () => ({ items }), - }; -} - -function item( +/** A 10 pt run starting at (x, y), 5 pt per character. */ +function run( str: string, x: number, y: number, - extra: Partial = {}, -): PdfTextItem { + extra: Partial = {}, +): TextItem { return { str, + dir: "ltr", transform: [10, 0, 0, 10, x, y], width: str.length * 5, height: 10, + fontName: "g_d0_f1", hasEOL: false, ...extra, }; } -describe("pageTextLayer", () => { - it("breaks once where pdf.js marks the end of a line", async () => { - const text = await pageTextLayer( - pageOf([ - item("Haemoglobin", 10, 700, { hasEOL: true }), - item("14.2 g/dL", 10, 688), +describe("joinTextRuns", () => { + it("keeps a word drawn in two fonts whole", () => { + expect( + joinTextRuns([ + run("Haem", 10, 700), + run("oglobin", 30, 700, { fontName: "g_d0_f2" }), ]), - ); - expect(text).toBe("Haemoglobin\n14.2 g/dL"); + ).toBe("Haemoglobin"); }); - it("breaks when the next item sits a line lower without a marked end", async () => { - const text = await pageTextLayer( - pageOf([item("Creatinine", 10, 700), item("0.9 mg/dL", 10, 680)]), + it("puts a space between runs on one line that leave a gap", () => { + expect(joinTextRuns([run("Glucose", 10, 700), run("92", 200, 700)])).toBe( + "Glucose 92", ); - expect(text).toBe("Creatinine\n0.9 mg/dL"); }); - it("does not break for a small baseline shift on one line", async () => { - const text = await pageTextLayer( - pageOf([item("10", 10, 700), item("3", 21, 703)]), - ); - expect(text).toBe("103"); + it("breaks once where a run ends its line", () => { + expect( + joinTextRuns([ + run("Haemoglobin", 10, 700, { hasEOL: true }), + run("14.2 g/dL", 10, 688), + ]), + ).toBe("Haemoglobin\n14.2 g/dL"); }); - it("separates far-apart items on one line with a tab", async () => { - const text = await pageTextLayer( - pageOf([item("Glucose", 10, 700), item("92", 200, 700)]), - ); - expect(text).toBe("Glucose\t92"); + it("breaks where the next run sits on another line without a marked end", () => { + expect( + joinTextRuns([run("Creatinine", 10, 700), run("0.9", 10, 680)]), + ).toBe("Creatinine\n0.9"); }); - it("skips marked-content items, which carry no text", async () => { - const text = await pageTextLayer( - pageOf([{ hasEOL: false } as PdfTextItem, item("Text", 10, 700)]), - ); - expect(text).toBe("Text"); + it("keeps a raised run on its line", () => { + expect(joinTextRuns([run("10", 10, 700), run("9", 20, 703)])).toBe("109"); + }); + + it("adds no blank line for an empty run that only ends a line", () => { + expect( + joinTextRuns([ + run("Line", 10, 700, { hasEOL: true }), + run("", 30, 700, { hasEOL: true }), + run("Next", 10, 688), + ]), + ).toBe("Line\nNext"); + }); + + it("skips marked-content markers, which carry no text", () => { + const marker: TextMarkedContent = { + type: "beginMarkedContent", + id: "mc0", + }; + expect(joinTextRuns([marker, run("Text", 10, 700)])).toBe("Text"); }); }); @@ -89,10 +98,10 @@ describe("readPdfText", () => { return Buffer.from(doc.output("arraybuffer")); } - it("joins pages with a blank line and adds no page markers", async () => { + it("reads the pages one after another and adds no page markers", async () => { const text = await readPdfText(pages(2)); expect(text).toBe( - "Page 1 lab report\nsecond line\n\nPage 2 lab report\nsecond line\n\n", + "Page 1 lab report\nsecond line\nPage 2 lab report\nsecond line", ); }); @@ -102,6 +111,17 @@ describe("readPdfText", () => { expect(text).not.toContain("Page 3"); }); + it("keeps a word whole that a real PDF draws in two fonts", async () => { + const doc = new jsPDF(); + doc.setFont("helvetica", "bold"); + doc.text("Haem", 10, 10); + const joint = 10 + doc.getTextWidth("Haem"); + doc.setFont("times", "normal"); + doc.text("oglobin 14.2 g/dL", joint, 10); + const text = await readPdfText(Buffer.from(doc.output("arraybuffer"))); + expect(text).toBe("Haemoglobin 14.2 g/dL"); + }); + it("throws what pdf.js throws on a file that is not a PDF", async () => { await expect(readPdfText(Buffer.from("not a pdf"))).rejects.toThrow(); }); diff --git a/src/lib/documents/local-extract.ts b/src/lib/documents/local-extract.ts index e726f4a27..327f8a8b9 100644 --- a/src/lib/documents/local-extract.ts +++ b/src/lib/documents/local-extract.ts @@ -7,13 +7,14 @@ * never the native `@napi-rs/canvas`), in milliseconds, with no OCR and no * third-party egress. * - * It reads through the app's own `pdfjs-dist`, the copy `./rasterize-pdf` - * renders with. It used to go through `pdf-parse`, which brings a second - * pdfjs of its own, and the two cannot share a process: both take their - * worker from `globalThis.pdfjsWorker`, so whichever ran first broke the - * other ("The API version "6.4.299" does not match the Worker version - * "5.4.296""). Thumbnails and vision reads failed after a text extraction, - * and the text extraction failed after a raster. + * It reads through `./pdfjs`, the one loader `./rasterize-pdf` opens documents + * with too, so both run the app's own `pdfjs-dist` with the same options. It + * used to go through `pdf-parse`, which brings a second pdfjs of its own, and + * the two cannot share a process: both take their worker from + * `globalThis.pdfjsWorker`, so whichever ran first broke the other ("The API + * version "6.4.299" does not match the Worker version "5.4.296""). Thumbnails + * and vision reads failed after a text extraction, and the text extraction + * failed after a raster. * The extracted text feeds the SAME blind, encrypted content index the provider * path writes (`upsertContentIndex`), so nothing readable is stored at rest. * @@ -28,92 +29,10 @@ */ import { Buffer } from "node:buffer"; -import type { PDFDocumentLoadingTask } from "pdfjs-dist"; -import type { - TextItem, - TextMarkedContent, -} from "pdfjs-dist/types/src/display/api"; - import { annotate } from "@/lib/logging/context"; +import { logCaught } from "@/lib/logging/signal"; -// pdfjs' module top-level references the browser-only `DOMMatrix` global. A -// STATIC import makes the Turbopack server chunk evaluate that reference at -// instantiation and throw `DOMMatrix is not defined`, taking down every -// route/worker that shares the chunk (the document routes among them). The -// module is therefore loaded lazily inside `extractPdfText` via a runtime -// `import()`. The type imports above are erased at build, so they load -// nothing; they let the compiler check this file against pdfjs' own API. - -/** - * Where a text run sits on the page. pdfjs gives each run a text matrix - * `[a, b, c, d, x, y]`: `x` and `y` are where the run starts, and `d` is its - * vertical scale, which for upright text is the font size. - */ -interface RunPosition { - x: number; - /** The baseline the run sits on. */ - y: number; - /** Where the run ends: its start plus its width. */ - end: number; - fontSize: number; -} - -function positionOf(run: TextItem): RunPosition { - const [, , , scaleY, x = 0, y = 0] = run.transform as number[]; - return { - x, - y, - end: x + run.width, - fontSize: Math.abs(scaleY ?? 0) || run.height || 1, - }; -} - -/** - * The next run is on a new line when its baseline moved by more than half a - * font size. Runs on one line can sit a little higher or lower (a superscript, - * a mixed font), so a small shift does not count. - */ -function startsNewLine(previous: RunPosition, next: RunPosition): boolean { - return Math.abs(next.y - previous.y) > previous.fontSize / 2; -} - -/** - * Two runs on one line are separate words when the gap between them is wider - * than a tenth of the font size. Runs that touch, such as a word drawn in two - * fonts, stay one word. - */ -function leavesWordGap(previous: RunPosition, next: RunPosition): boolean { - return next.x - previous.end > next.fontSize / 10; -} - -/** - * One page's text: its runs in order, with a line break where a run ends a - * line or the next one starts a new line, and a space where two runs on one - * line leave a gap between words. - */ -function pageText(items: Array): string { - let text = ""; - let previous: RunPosition | null = null; - - for (const item of items) { - // Marked-content markers carry no text. - if (!("str" in item)) continue; - const position = positionOf(item); - - if (previous && !text.endsWith("\n")) { - if (startsNewLine(previous, position)) { - text += "\n"; - } else if (leavesWordGap(previous, position) && !/\s$/.test(text)) { - text += " "; - } - } - text += item.str; - if (item.hasEOL && !text.endsWith("\n")) text += "\n"; - - previous = position; - } - return text; -} +import { readPdfText } from "./pdfjs"; /** * Minimum joined text length for a PDF to count as a real text-layer document. @@ -145,49 +64,32 @@ export type LocalExtractResult = * Extract a text-layer PDF's embedded text server-side. Returns `empty` for a * scanned/no-text-layer PDF and `error` for a malformed/encrypted one — never * throws, so a bad document can never abort the caller (the upload/backfill job - * just leaves it un-indexed). + * just leaves it un-indexed). An `error` leaves a `warn` line naming what + * pdf.js threw, because the outcome alone does not say why a document stayed + * out of the index. */ export async function extractPdfText( buffer: Buffer, ): Promise { - let task: PDFDocumentLoadingTask | null = null; + let text: string; try { - // Lazy load: keeps pdfjs/DOMMatrix out of the eager server chunk (see the - // import-site note). Resolved at first PDF extraction, then module-cached. - const pdfjs = await import("pdfjs-dist/legacy/build/pdf.mjs"); - task = pdfjs.getDocument({ - data: new Uint8Array(buffer), - verbosity: 0, // VerbosityLevel.ERRORS — no console spam on odd PDFs. - useSystemFonts: false, - }); - const doc = await task.promise; // Bound the page walk so a pathological many-page PDF costs no more. - const pageCount = Math.min(doc.numPages, LOCAL_MAX_PAGES); - const pages: string[] = []; - for (let n = 1; n <= pageCount; n++) { - const page = await doc.getPage(n); - try { - pages.push(pageText((await page.getTextContent()).items)); - } finally { - page.cleanup(); - } - } - const text = pages.join("\n").trim(); - if (text.length < LOCAL_TEXT_MIN_CHARS) { - return { ok: false, reason: "empty" }; - } - return { ok: true, text, source: "local-pdf" }; - } catch { + text = (await readPdfText(buffer, LOCAL_MAX_PAGES)).trim(); + } catch (err) { + const errorType = err instanceof Error ? err.name || "Error" : typeof err; + annotate({ + action: { name: "documents.localIndex.extractFailed" }, + meta: { localExtractError: errorType }, + }); + logCaught("documents.local_extract.failed", err, { + bytes: buffer.length, + }); return { ok: false, reason: "error" }; - } finally { - if (task) { - try { - await task.destroy(); - } catch { - // Best-effort teardown; a destroy failure must not mask the result. - } - } } + if (text.length < LOCAL_TEXT_MIN_CHARS) { + return { ok: false, reason: "empty" }; + } + return { ok: true, text, source: "local-pdf" }; } /** diff --git a/src/lib/documents/pdfjs.ts b/src/lib/documents/pdfjs.ts index 5d44c1f31..9409323e9 100644 --- a/src/lib/documents/pdfjs.ts +++ b/src/lib/documents/pdfjs.ts @@ -1,5 +1,6 @@ /** - * The one pdf.js this server runs, and the one way it is opened. + * The one pdf.js this server runs, the one way it is opened, and the one way + * its text layer is read. * * Under Node, pdf.js does not start a worker thread: it loads its worker code * into the same process and keeps it on `globalThis.pdfjsWorker`. Whichever @@ -15,7 +16,9 @@ * reads the browser-only `DOMMatrix` global, so a static import would make the * server chunk evaluate it at instantiation and throw `DOMMatrix is not * defined`, taking down every route and worker sharing the chunk. The import - * resolves once and is module-cached after that. + * resolves once and is module-cached after that. The type imports below are + * erased at build, so they load nothing; they let the compiler check every + * call here against pdf.js' own API. * * This module has no logging of its own, so the end-to-end suite can read * report text with it; the callers say what failed. @@ -25,74 +28,11 @@ import { existsSync } from "node:fs"; import { createRequire } from "node:module"; import { dirname, join } from "node:path"; -// Structural types for the slice of the pdf.js API used here and in the -// rasterizer. Type-only, so nothing evaluates pdf.js for its types. -export interface PdfViewport { - width: number; - height: number; - convertToViewportPoint(x: number, y: number): number[]; -} -export interface PdfCanvas { - width: number; - height: number; - toBuffer(mime: "image/jpeg", quality?: number): Buffer; -} -export interface PdfContext2D { - getImageData( - x: number, - y: number, - width: number, - height: number, - ): { data: Uint8ClampedArray }; -} -interface PdfCanvasAndContext { - canvas: PdfCanvas; - context: PdfContext2D; -} -interface PdfCanvasFactory { - create(width: number, height: number): PdfCanvasAndContext; - destroy?(target: PdfCanvasAndContext): void; -} -/** A text-layer item; marked-content items carry no `str` and are skipped. */ -export interface PdfTextItem { - str?: string; - hasEOL?: boolean; - transform?: number[]; - width?: number; - height?: number; -} -export interface PdfPage { - getViewport(opts: { scale: number }): PdfViewport; - render(opts: { - canvasContext: unknown; - viewport: PdfViewport; - canvas: unknown; - }): { promise: Promise }; - cleanup(): void; - getTextContent(): Promise<{ items: PdfTextItem[] }>; -} -export interface PdfDocument { - numPages: number; - canvasFactory: PdfCanvasFactory; - getPage(pageNumber: number): Promise; -} -export interface PdfLoadingTask { - promise: Promise; - destroy(): Promise; -} -interface PdfDocumentOptions { - data: Uint8Array; - verbosity?: number; - isEvalSupported?: boolean; - useSystemFonts?: boolean; - standardFontDataUrl?: string; - cMapUrl?: string; - cMapPacked?: boolean; - wasmUrl?: string; -} -interface PdfjsModule { - getDocument(opts: PdfDocumentOptions): PdfLoadingTask; -} +import type { PDFDocumentLoadingTask } from "pdfjs-dist"; +import type { + TextItem, + TextMarkedContent, +} from "pdfjs-dist/types/src/display/api"; /** * Where pdf.js finds the data it does not carry in its JS: the standard 14 @@ -169,15 +109,13 @@ export function __resetPdfjsAssetsForTests(): void { */ export async function openPdfDocument( data: Uint8Array | Buffer, -): Promise { - const pdfjs = - (await import("pdfjs-dist/legacy/build/pdf.mjs")) as unknown as PdfjsModule; +): Promise { + const pdfjs = await import("pdfjs-dist/legacy/build/pdf.mjs"); const assets = resolvePdfjsAssets(); return pdfjs.getDocument({ // A copy: pdf.js transfers the buffer it is given to its worker. data: new Uint8Array(data), verbosity: 0, // VerbosityLevel.ERRORS: no console noise on odd PDFs. - isEvalSupported: false, // never eval font programs (defence in depth). // There are no system fonts to use: substitutes come from pdf.js' own // standard font data (and, when rendering, the registered fallbacks). useSystemFonts: false, @@ -192,73 +130,83 @@ export async function openPdfDocument( }); } -/** Vertical gap (pt) past which the next item starts a new line. */ -const LINE_THRESHOLD = 4.6; -/** Horizontal gap (pt) on one line past which the next item is a new cell. */ -const CELL_THRESHOLD = 7; - /** - * One page's text layer as plain text: items in content order, a line break - * where pdf.js marks one or where the next item sits on a lower line, and a - * tab between items on one line that are far apart (table cells). - * - * This is the layout `pdf-parse` produced with its defaults, kept so that - * swapping it out did not change what the content index sees. + * Where a text run sits on the page. pdfjs gives each run a text matrix + * `[a, b, c, d, x, y]`: `x` and `y` are where the run starts, and `d` is its + * vertical scale, which for upright text is the font size. */ -export async function pageTextLayer( - page: Pick, -): Promise { - const viewport = page.getViewport({ scale: 1 }); - const { items } = await page.getTextContent(); - const parts: string[] = []; - let lastX: number | undefined; - let lastY: number | undefined; - let lineHeight = 0; - for (const item of items) { - if (typeof item.str !== "string") continue; - const transform = item.transform ?? [1, 0, 0, 1, 0, 0]; - const [x = 0, y = 0] = viewport.convertToViewportPoint( - transform[4] ?? 0, - transform[5] ?? 0, - ); - let str = item.str; - if (lastY !== undefined) { - const gap = Math.abs(lastY - y); - if (gap > LINE_THRESHOLD) { - const previous = parts.at(-1); - const opensLine = - str.startsWith("\n") || (str.trim() === "" && item.hasEOL === true); - if ( - previous !== undefined && - !previous.endsWith("\n") && - !opensLine && - gap - 1 > lineHeight - ) { - parts.push("\n"); - lineHeight = 0; - } - } else if ( - gap < LINE_THRESHOLD && - lastX !== undefined && - Math.abs(lastX - x) > CELL_THRESHOLD - ) { - str = `\t${str}`; - } - } - parts.push(str); - lastX = x + (item.width ?? 0); - lastY = y; - lineHeight = Math.max(lineHeight, item.height ?? 0); - if (item.hasEOL) parts.push("\n"); - if (item.hasEOL || str.endsWith("\n")) lineHeight = 0; - } - return parts.join(""); +interface RunPosition { + x: number; + /** The baseline the run sits on. */ + y: number; + /** Where the run ends: its start plus its width. */ + end: number; + fontSize: number; +} + +function positionOf(run: TextItem): RunPosition { + const [, , , scaleY, x = 0, y = 0] = run.transform as number[]; + return { + x, + y, + end: x + run.width, + fontSize: Math.abs(scaleY ?? 0) || run.height || 1, + }; } /** - * The text layer of the first `maxPages` pages, each page followed by a blank - * line. Throws what pdf.js throws (a malformed file, a password, a page that - * will not parse); the document is torn down either way. + * The next run is on a new line when its baseline moved by more than half a + * font size. Runs on one line can sit a little higher or lower (a superscript, + * a mixed font), so a small shift does not count. + */ +function startsNewLine(previous: RunPosition, next: RunPosition): boolean { + return Math.abs(next.y - previous.y) > previous.fontSize / 2; +} + +/** + * Two runs on one line are separate words when the gap between them is wider + * than a tenth of the font size. Runs that touch, such as a word drawn in two + * fonts, stay one word. + */ +function leavesWordGap(previous: RunPosition, next: RunPosition): boolean { + return next.x - previous.end > next.fontSize / 10; +} + +/** + * One page's text: its runs in order, with a line break where a run ends a + * line or the next one starts a new line, and a space where two runs on one + * line leave a gap between words. + */ +export function joinTextRuns( + items: Array, +): string { + let text = ""; + let previous: RunPosition | null = null; + + for (const item of items) { + // Marked-content markers carry no text. + if (!("str" in item)) continue; + const position = positionOf(item); + + if (previous && !text.endsWith("\n")) { + if (startsNewLine(previous, position)) { + text += "\n"; + } else if (leavesWordGap(previous, position) && !/\s$/.test(text)) { + text += " "; + } + } + text += item.str; + if (item.hasEOL && !text.endsWith("\n")) text += "\n"; + + previous = position; + } + return text; +} + +/** + * The text layer of the first `maxPages` pages, one page after another. + * Throws what pdf.js throws (a malformed file, a password, a page that will + * not parse); the document is torn down either way. */ export async function readPdfText( data: Uint8Array | Buffer, @@ -267,17 +215,17 @@ export async function readPdfText( const task = await openPdfDocument(data); try { const doc = await task.promise; - const last = Math.min(doc.numPages, maxPages); - let text = ""; - for (let pageNumber = 1; pageNumber <= last; pageNumber++) { - const page = await doc.getPage(pageNumber); + const pageCount = Math.min(doc.numPages, maxPages); + const pages: string[] = []; + for (let n = 1; n <= pageCount; n++) { + const page = await doc.getPage(n); try { - text += `${await pageTextLayer(page)}\n\n`; + pages.push(joinTextRuns((await page.getTextContent()).items)); } finally { page.cleanup(); } } - return text; + return pages.join("\n"); } finally { try { await task.destroy(); diff --git a/src/lib/documents/rasterize-pdf.ts b/src/lib/documents/rasterize-pdf.ts index f23730e14..5bb98f32a 100644 --- a/src/lib/documents/rasterize-pdf.ts +++ b/src/lib/documents/rasterize-pdf.ts @@ -33,15 +33,7 @@ import { Buffer } from "node:buffer"; import { annotate } from "@/lib/logging/context"; import { nativeCanvasSupported } from "./native-canvas-support"; -import { - openPdfDocument, - resolvePdfjsAssets, - type PdfContext2D, - type PdfDocument, - type PdfjsAssets, - type PdfLoadingTask, - type PdfPage, -} from "./pdfjs"; +import { openPdfDocument, resolvePdfjsAssets, type PdfjsAssets } from "./pdfjs"; /** * Whether the PDF rasterizer is available in this build. The native @@ -114,6 +106,33 @@ export type RasterResult = } | { ok: false; reason: "unsupported" | "render-failed" }; +/** + * The canvas pdf.js draws into under Node is `@napi-rs/canvas`. pdf.js' own + * declarations are written for the DOM and type the document's canvas factory + * as a bare `object`, so the slice used here is named: a DOM-shaped canvas + * and 2D context, plus the JPEG encoder the native canvas adds. + */ +interface RenderTarget { + canvas: HTMLCanvasElement & { + toBuffer(mime: "image/jpeg", quality?: number): Buffer; + }; + context: CanvasRenderingContext2D; +} +interface RenderTargetFactory { + create(width: number, height: number): RenderTarget; + destroy?(target: RenderTarget): void; +} + +/** + * A page's text layer, as far as `pageText` reads it: text runs, and + * marked-content markers that carry only a `type` and no text. + */ +interface TextLayerPage { + getTextContent(): Promise<{ + items: Array<{ str?: string; hasEOL?: boolean; type?: string }>; + }>; +} + interface GlobalFontsLike { registerFromPath(path: string, nameAlias?: string): unknown; } @@ -144,7 +163,11 @@ async function registerFallbackFamilies(assets: PdfjsAssets): Promise { } /** Share of dark pixels on a rendered page (luminance below half). */ -function inkRatio(context: PdfContext2D, width: number, height: number) { +function inkRatio( + context: Pick, + width: number, + height: number, +) { const { data } = context.getImageData(0, 0, width, height); let dark = 0; for (let i = 0; i < data.length; i += 4) { @@ -188,9 +211,7 @@ function headThatFits(text: string, room: number): string { * `RASTER_PAGE_TEXT_MAX` it is cut (at a line, else a space, else hard) and * marked. */ -export async function pageText( - page: Pick, -): Promise { +export async function pageText(page: TextLayerPage): Promise { const max = RASTER_PAGE_TEXT_MAX; try { const { items } = await page.getTextContent(); @@ -248,8 +269,7 @@ export async function rasterizePdf( maxPages: number = RASTER_MAX_PAGES, opts: { withText?: boolean } = {}, ): Promise { - let doc: PdfDocument | null = null; - let task: PdfLoadingTask | null = null; + let task: Awaited> | null = null; // CPU-feature gate: pdfjs renders through @napi-rs/canvas (Skia), whose x64 // build uses AVX2 — an unsupported CPU dies with an uncatchable SIGILL on // the first render. Degrade to the existing no-raster path instead (scanned @@ -273,7 +293,7 @@ export async function rasterizePdf( } task = await openPdfDocument(buffer); - doc = await task.promise; + const doc = await task.promise; // Never exceed the standing DoS ceiling, even if a caller asks for more. const cap = Math.min(Math.max(1, maxPages), RASTER_HARD_MAX_PAGES); @@ -292,7 +312,7 @@ export async function rasterizePdf( longEdge > 0 ? Math.min(1, RASTER_TARGET_LONG_EDGE / longEdge) : 1; const viewport = page.getViewport({ scale }); - const factory = doc.canvasFactory; + const factory = doc.canvasFactory as RenderTargetFactory; const cc = factory.create( Math.ceil(viewport.width), Math.ceil(viewport.height), From 1fd34e4d749f6642abbc6c4bb8a95b26a88d0597 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Sat, 24 Oct 2026 20:16:02 +0200 Subject: [PATCH 08/11] test(documents): keep one pair of one-process order tests Two pairs of tests ran a text read and a rasterization in one process, one file per order. The pdfjs-one-version pair stays; the other pair and its shared helper go. The kept pair now first asserts that nothing in the process has claimed the pdf.js worker yet, so a file that lands in a shared process fails instead of passing on an order it never ran, and it mocks the failure signal the text read can emit. Refs #1277 --- .../pdfjs-one-runtime-raster-first.test.ts | 26 ---------- .../pdfjs-one-runtime-text-first.test.ts | 26 ---------- .../documents/__tests__/pdfjs-one-runtime.ts | 50 ------------------- .../pdfjs-one-version-extract-first.test.ts | 7 +++ .../pdfjs-one-version-raster-first.test.ts | 7 +++ 5 files changed, 14 insertions(+), 102 deletions(-) delete mode 100644 src/lib/documents/__tests__/pdfjs-one-runtime-raster-first.test.ts delete mode 100644 src/lib/documents/__tests__/pdfjs-one-runtime-text-first.test.ts delete mode 100644 src/lib/documents/__tests__/pdfjs-one-runtime.ts diff --git a/src/lib/documents/__tests__/pdfjs-one-runtime-raster-first.test.ts b/src/lib/documents/__tests__/pdfjs-one-runtime-raster-first.test.ts deleted file mode 100644 index 1af8dc44f..000000000 --- a/src/lib/documents/__tests__/pdfjs-one-runtime-raster-first.test.ts +++ /dev/null @@ -1,26 +0,0 @@ -import { describe, it, vi } from "vitest"; - -/** - * GitHub #1277, rasterization first: a render used to claim the in-process - * pdf.js worker for its own copy, and every later local text read failed - * without a word, leaving the document out of the search index. See - * `pdfjs-one-runtime.ts`. - */ - -vi.mock("@/lib/logging/context", () => ({ annotate: vi.fn() })); -vi.mock("@/lib/logging/signal", () => ({ logCaught: vi.fn() })); - -import { - expectFreshProcess, - expectRaster, - expectTextRead, -} from "./pdfjs-one-runtime"; - -describe("one pdf.js per process (#1277): rasterization first", () => { - it("reads text after a rasterization, and rasterizes again after that", async () => { - expectFreshProcess(); - await expectRaster(); - await expectTextRead(); - await expectRaster(); - }); -}); diff --git a/src/lib/documents/__tests__/pdfjs-one-runtime-text-first.test.ts b/src/lib/documents/__tests__/pdfjs-one-runtime-text-first.test.ts deleted file mode 100644 index 846ec1852..000000000 --- a/src/lib/documents/__tests__/pdfjs-one-runtime-text-first.test.ts +++ /dev/null @@ -1,26 +0,0 @@ -import { describe, it, vi } from "vitest"; - -/** - * GitHub #1277, text first: a local text read used to claim the in-process - * pdf.js worker for its own copy, and every later rasterization (thumbnails, - * the vision read of a scan) failed until a restart. See - * `pdfjs-one-runtime.ts`. - */ - -vi.mock("@/lib/logging/context", () => ({ annotate: vi.fn() })); -vi.mock("@/lib/logging/signal", () => ({ logCaught: vi.fn() })); - -import { - expectFreshProcess, - expectRaster, - expectTextRead, -} from "./pdfjs-one-runtime"; - -describe("one pdf.js per process (#1277): text read first", () => { - it("rasterizes after a text read, and reads text again after that", async () => { - expectFreshProcess(); - await expectTextRead(); - await expectRaster(); - await expectTextRead(); - }); -}); diff --git a/src/lib/documents/__tests__/pdfjs-one-runtime.ts b/src/lib/documents/__tests__/pdfjs-one-runtime.ts deleted file mode 100644 index d3fb7a672..000000000 --- a/src/lib/documents/__tests__/pdfjs-one-runtime.ts +++ /dev/null @@ -1,50 +0,0 @@ -/** - * Shared steps for the two `pdfjs-one-runtime-*` files (GitHub #1277). - * - * Under Node, pdf.js runs its worker in-process and keeps it on - * `globalThis.pdfjsWorker`. Whichever pdf.js copy opens a document first - * owns that global for the life of the process, and a different copy then - * refuses every document with "The API version … does not match the Worker - * version …". The text reader and the rasterizer once loaded different - * copies, so whichever ran first broke the other until a restart. - * - * Each order needs a fresh process, because the global cannot be taken back - * once claimed; that is why the two orders live in two files (the isolated - * project gives each file its own process) instead of two tests here. - */ -import { readFileSync } from "node:fs"; -import { join } from "node:path"; - -import { expect } from "vitest"; - -import { extractPdfText } from "../local-extract"; -import { rasterizePdf } from "../rasterize-pdf"; - -const FIXTURE = join( - process.cwd(), - "tests", - "fixtures", - "pdf", - "fallback-fonts.pdf", -); - -const pdf = () => readFileSync(FIXTURE); - -/** Nothing in this process has opened a PDF yet. */ -export function expectFreshProcess(): void { - expect( - (globalThis as { pdfjsWorker?: unknown }).pdfjsWorker, - "the file must run in its own process; another file already opened a PDF", - ).toBeUndefined(); -} - -export async function expectTextRead(): Promise { - const result = await extractPdfText(pdf()); - expect(result).toMatchObject({ ok: true, source: "local-pdf" }); - if (result.ok) expect(result.text).toContain("Test Insurance AG"); -} - -export async function expectRaster(): Promise { - const result = await rasterizePdf(pdf(), 1); - expect(result).toMatchObject({ ok: true }); -} diff --git a/src/lib/documents/__tests__/pdfjs-one-version-extract-first.test.ts b/src/lib/documents/__tests__/pdfjs-one-version-extract-first.test.ts index 298696510..596523e00 100644 --- a/src/lib/documents/__tests__/pdfjs-one-version-extract-first.test.ts +++ b/src/lib/documents/__tests__/pdfjs-one-version-extract-first.test.ts @@ -21,6 +21,7 @@ import { describe, expect, it, vi } from "vitest"; * `extractPdfText` → the raster step here fails. */ vi.mock("@/lib/logging/context", () => ({ annotate: vi.fn() })); +vi.mock("@/lib/logging/signal", () => ({ logCaught: vi.fn() })); import { extractPdfText } from "../local-extract"; import { rasterizePdf } from "../rasterize-pdf"; @@ -31,6 +32,12 @@ const fixture = readFileSync( describe("one pdfjs for text and raster: text first", () => { it("extracts, rasterises and extracts again in one process", async () => { + // The order only means something in a process that has not opened a PDF. + expect( + (globalThis as { pdfjsWorker?: unknown }).pdfjsWorker, + "this file must run in a process of its own", + ).toBeUndefined(); + const first = await extractPdfText(fixture); expect(first.ok).toBe(true); diff --git a/src/lib/documents/__tests__/pdfjs-one-version-raster-first.test.ts b/src/lib/documents/__tests__/pdfjs-one-version-raster-first.test.ts index 3389f078e..94b987be7 100644 --- a/src/lib/documents/__tests__/pdfjs-one-version-raster-first.test.ts +++ b/src/lib/documents/__tests__/pdfjs-one-version-raster-first.test.ts @@ -21,6 +21,7 @@ import { describe, expect, it, vi } from "vitest"; * `extractPdfText` → the text step here fails. */ vi.mock("@/lib/logging/context", () => ({ annotate: vi.fn() })); +vi.mock("@/lib/logging/signal", () => ({ logCaught: vi.fn() })); import { extractPdfText } from "../local-extract"; import { rasterizePdf } from "../rasterize-pdf"; @@ -31,6 +32,12 @@ const fixture = readFileSync( describe("one pdfjs for text and raster: raster first", () => { it("rasterises, extracts and rasterises again in one process", async () => { + // The order only means something in a process that has not opened a PDF. + expect( + (globalThis as { pdfjsWorker?: unknown }).pdfjsWorker, + "this file must run in a process of its own", + ).toBeUndefined(); + const first = await rasterizePdf(fixture, 1); expect(first.ok).toBe(true); From b716ee590bed99e58bc198cd097ec39b06915db6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Sat, 24 Oct 2026 20:16:07 +0200 Subject: [PATCH 09/11] test: fold the two one-pdf.js guards into one Two guards with the same file name froze the same failure from two sides: one stopped a runtime import of pdf-parse, the other pinned one pdfjs-dist in package.json, the lockfile and the checkout, and allowed only the shared loader to import it. They are one guard now. It strips comments before it matches, so prose naming a package does not count, walks every runtime extension and skips __mocks__ as well as __tests__, treats a type-only import of pdfjs-dist outside the loader as an import, and keeps the import shapes each side's self-test proved, including the near-miss name that must not match. Refs #1277 --- src/__tests__/single-pdfjs-guard.test.ts | 228 +++++++++++++++++++---- 1 file changed, 187 insertions(+), 41 deletions(-) diff --git a/src/__tests__/single-pdfjs-guard.test.ts b/src/__tests__/single-pdfjs-guard.test.ts index d42270995..6f7d67b77 100644 --- a/src/__tests__/single-pdfjs-guard.test.ts +++ b/src/__tests__/single-pdfjs-guard.test.ts @@ -1,67 +1,196 @@ /** - * Structural guard: the server runs one pdfjs. + * Structural guard: the server runs one pdf.js, and only one module opens it. * - * ## The failure this freezes + * ## The failure this freezes (GitHub #1277) * - * `pdf-parse` brings its own `pdfjs-dist` beside the app's, and under Node - * every pdfjs copy keeps its worker on the one `globalThis.pdfjsWorker`. While - * local text extraction went through pdf-parse, whichever copy ran first in a - * process broke the other: "The API version "6.4.299" does not match the - * Worker version "5.4.296"". Thumbnails and vision reads failed after a text - * extraction, and the text extraction failed after a raster. + * Under Node, pdf.js keeps its in-process worker on `globalThis.pdfjsWorker`, + * and whichever copy opens a document first owns it until the process exits. + * Local text extraction went through `pdf-parse`, which pinned its own, older + * pdfjs-dist, while the rasterizer ran the app's. Whichever ran first broke + * the other with "The API version "6.4.299" does not match the Worker version + * "5.4.296"": after a text extraction every thumbnail and every vision read + * of a scan failed, after a raster every local text extraction failed without + * a word, until a restart. Nothing in the gate noticed, because each path + * passed on its own and the image tracing shipped every `pdfjs-dist@*` the + * store held. * * ## What it proves and what it does not * - * No runtime file under `src/` imports `pdf-parse`, so the only pdfjs the - * server loads is the app's own. Tests may still read generated PDFs back with - * it; they run in their own processes. The pdfjs-one-version tests prove the - * two orders in one process; this file stops the import from coming back. + * - package.json pins pdfjs-dist exactly and lists no pdf-parse. + * - The lockfile resolves exactly one pdfjs-dist, the version package.json + * pins, and no package in it depends on pdfjs-dist except the app itself. + * A new dependency that brings its own pdf.js, even the same version, + * fails here and has to be looked at; every line of the lockfile that + * names pdfjs-dist is either understood or reported. + * - The checkout resolves that same version from the working directory, + * which is how the server finds it. + * - No runtime module under `src/` imports pdf-parse, and none imports + * pdfjs-dist (a type-only import included) except + * `src/lib/documents/pdfjs.ts`, so the text read and the rasterizer open + * documents with the same options. Comments are stripped first, so prose + * that names a package does not count. + * + * It does not prove the image carries one copy; the Dockerfile hoists the + * pinned version by name and the tracing glob can only find what the lockfile + * installed. The behaviour itself is pinned by the two `pdfjs-one-version-*` + * tests under `src/lib/documents/__tests__/`, one per order, each in a + * process of its own. */ +import { createRequire } from "node:module"; import { join } from "node:path"; import { describe, expect, it } from "vitest"; import { + readSourceFolder, readSourceText, - requireFloor, stripComments, - walkSourceFiles, } from "./helpers/source-files"; -const SRC = join(process.cwd(), "src"); +const ROOT = process.cwd(); +/** Relative to `src/`. */ +const LOADER = "lib/documents/pdfjs.ts"; -/** - * Every runtime module under `src/`: the suites, mocks and generated client - * are not loaded by the server. About 3,400 today; the floor sits below that - * with headroom, so a walk that comes back short fails instead of passing. - */ -function runtimeFiles(): string[] { - const files = walkSourceFiles(SRC, { - floor: 5000, - extensions: [".ts", ".tsx", ".js", ".jsx", ".mjs", ".cjs", ".mts", ".cts"], - }).filter( - (rel) => - !rel.startsWith("generated/") && - !rel.split("/").includes("__tests__") && - !rel.split("/").includes("__mocks__") && - !/\.test\.(ts|tsx)$/.test(rel), - ); - return [...requireFloor("runtime files under src/", files, 3000)]; +const pkg = JSON.parse(readSourceText(join(ROOT, "package.json"))) as { + dependencies?: Record; + devDependencies?: Record; +}; +const pinned = + pkg.dependencies?.["pdfjs-dist"] ?? pkg.devDependencies?.["pdfjs-dist"]; + +/** Every lockfile line naming pdfjs-dist, sorted into what it means. */ +function readLockfile(lockfile: string): { + versions: Set; + rootImporters: number; + unexplained: string[]; +} { + const lines = lockfile.split("\n"); + const versions = new Set(); + let rootImporters = 0; + const unexplained: string[] = []; + lines.forEach((line, i) => { + if (!line.includes("pdfjs-dist")) return; + // A resolved copy: ` pdfjs-dist@6.4.299:` in `packages:` and again in + // `snapshots:` (where a peer suffix in parentheses may follow). + const resolved = + /^ {2}'?pdfjs-dist@(\d+\.\d+\.\d+[\w.+-]*)(?:\(.*\))?'?:/.exec(line); + if (resolved) { + versions.add(resolved[1]!); + return; + } + // The app's own entry under `importers:`, followed by its specifier. + if ( + /^ {6}pdfjs-dist:\s*$/.test(line) && + /^\s+specifier:/.test(lines[i + 1] ?? "") + ) { + rootImporters++; + return; + } + // Anything else is another package depending on pdfjs-dist, an + // override, or a spelling this reader does not know: all reported. + unexplained.push(`${i + 1}: ${line.trim()}`); + }); + return { versions, rootImporters, unexplained }; } -/** A static import, a re-export, a dynamic import or a require of pdf-parse. */ -const PDF_PARSE_IMPORT = - /(?:\bfrom\s*|\bimport\s*\(\s*|\brequire\s*\(\s*|\bimport\s+)["']pdf-parse(?:\/[^"']*)?["']/; +/** A static import, a re-export, a dynamic import or a require of `name`. */ +function importOf(name: string): RegExp { + return new RegExp( + String.raw`(?:\bfrom\s*|\bimport\s*\(\s*|\brequire\s*\(\s*|\bimport\s+)["']` + + name + + String.raw`(?:/[^"']*)?["']`, + ); +} +const PDFJS_IMPORT = importOf("pdfjs-dist"); +const PDF_PARSE_IMPORT = importOf("pdf-parse"); -describe("one pdfjs in the server", () => { - it("no runtime file imports pdf-parse", () => { - const importers = runtimeFiles().filter((rel) => - PDF_PARSE_IMPORT.test(stripComments(readSourceText(join(SRC, rel)))), +/** The modules whose comment-free source matches `pattern`, sorted. */ +function importers(files: Map, pattern: RegExp): string[] { + return [...files] + .filter(([, source]) => pattern.test(source)) + .map(([path]) => path) + .sort(); +} + +/** + * Runtime modules under `src/`, keyed by their path relative to it, with + * comments stripped. The suites, mocks and generated client are not loaded + * by the server. + */ +function runtimeSources(): Map { + return new Map( + readSourceFolder("src", { + floor: 1000, + extensions: [ + ".ts", + ".tsx", + ".mts", + ".cts", + ".js", + ".jsx", + ".mjs", + ".cjs", + ], + include: (rel) => + !rel.startsWith("generated/") && + !rel.split("/").includes("__tests__") && + !rel.split("/").includes("__mocks__") && + !/\.(?:test|spec)\.[mc]?[tj]sx?$/.test(rel), + }).map(([rel, source]) => [rel, stripComments(source)]), + ); +} + +describe("one pdf.js (#1277)", () => { + it("package.json pins pdfjs-dist exactly and lists no pdf-parse", () => { + expect(pinned, "pdfjs-dist is a direct dependency").toMatch( + /^\d+\.\d+\.\d+$/, ); - expect(importers).toEqual([]); + expect(pkg.dependencies?.["pdf-parse"]).toBeUndefined(); + expect(pkg.devDependencies?.["pdf-parse"]).toBeUndefined(); }); - it("recognises each way of importing it", () => { + it("the lockfile resolves one pdfjs-dist, and only the app depends on it", () => { + const lock = readLockfile(readSourceText(join(ROOT, "pnpm-lock.yaml"))); + expect(lock.unexplained, "lockfile lines naming pdfjs-dist").toEqual([]); + expect([...lock.versions]).toEqual([pinned]); + expect(lock.rootImporters).toBe(1); + }); + + it("the checkout resolves the pinned copy", () => { + const anchor = createRequire(join(ROOT, "package.json")); + const resolved = JSON.parse( + readSourceText(anchor.resolve("pdfjs-dist/package.json")), + ) as { version: string }; + expect(resolved.version).toBe(pinned); + }); + + it("no runtime module imports pdf-parse, and only the shared loader imports pdf.js", () => { + const sources = runtimeSources(); + expect(sources.has(LOADER), `${LOADER} exists`).toBe(true); + expect(importers(sources, PDF_PARSE_IMPORT)).toEqual([]); + expect(importers(sources, PDFJS_IMPORT)).toEqual([LOADER]); + }); + + it("the readers catch what they exist to catch", () => { + const twoCopies = [ + "importers:", + " .:", + " devDependencies:", + " pdfjs-dist:", + " specifier: 6.4.299", + " version: 6.4.299", + "packages:", + " pdfjs-dist@5.4.296:", + " pdfjs-dist@6.4.299:", + "snapshots:", + " some-parser@2.4.5:", + " dependencies:", + " pdfjs-dist: 5.4.296", + ].join("\n"); + const lock = readLockfile(twoCopies); + expect([...lock.versions].sort()).toEqual(["5.4.296", "6.4.299"]); + expect(lock.unexplained).toEqual(["13: pdfjs-dist: 5.4.296"]); + for (const line of [ 'import { PDFParse } from "pdf-parse";', 'import type { PDFParse } from "pdf-parse";', @@ -72,5 +201,22 @@ describe("one pdfjs in the server", () => { expect(PDF_PARSE_IMPORT.test(line), line).toBe(true); } expect(PDF_PARSE_IMPORT.test('import x from "pdf-parser";')).toBe(false); + + const sources = new Map( + [ + [LOADER, 'await import("pdfjs-dist/legacy/build/pdf.mjs")'], + [ + "a", + 'import type { TextItem } from "pdfjs-dist/types/src/display/api";', + ], + ["b", "const m = await import(\n 'pdfjs-dist'\n);"], + ["c", 'import "pdfjs-dist/legacy/build/pdf.worker.mjs";'], + ["d", '// once read with import("pdfjs-dist") here'], + ["e", 'const name = "pdfjs-dist";'], + ].map(([rel, source]) => [rel!, stripComments(source!)]), + ); + expect(importers(sources, PDFJS_IMPORT)).toEqual( + ["a", "b", "c", LOADER].sort(), + ); }); }); From 4752115d7d24331a82685b8ca5f48ed060329d82 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Sat, 24 Oct 2026 20:51:28 +0200 Subject: [PATCH 10/11] feat(documents): add Recently deleted for deleted documents Deleting a document only stamps a tombstone and the purge removes it after 30 days, but the only way back was the undo toast, gone a few seconds after the delete. Recently deleted lists the deleted documents of the record with type, filing date, where an import came from, when each was deleted and how long it has left, and offers Restore, Restore all and Delete now. GET /api/documents/inbound/deleted lists them (owner-only, capped at 200 with the list-cap meta). POST /api/documents/inbound/{id}/purge removes one deleted document for good through the same function the nightly purge runs, so its import keys and aliases still move into the ledger and an importer re-sending it is still answered "deleted"; a live document answers 409 documents.inbound.notDeleted. The usage read gains deletedCount, 0 under a grant. The restore route and bulk restore share one restore, which now also clears ledger entries for the document's own keys. The vault header shows a quiet "Recently deleted (n)" link only while something is in it, and the delete toasts and confirmations say that deleted documents stay there for 30 days. The purge's queue facts and grace window move to document-purge-queue.ts so the worker registry still loads none of the work. Refs #1275 --- docs/api/openapi.yaml | 213 ++++++++++- messages/de.json | 29 +- messages/en.json | 29 +- messages/es.json | 29 +- messages/fr.json | 29 +- messages/it.json | 29 +- messages/ko.json | 29 +- messages/nl.json | 29 +- messages/pl.json | 29 +- messages/pt-BR.json | 29 +- .../baselines/source-file-lines.json | 4 +- .../baselines/source-function-lines.json | 5 +- .../destructive-control-guard.test.ts | 10 +- .../success-affordance-guard.test.ts | 7 + .../api/documents/inbound/[id]/purge/route.ts | 105 +++++ .../documents/inbound/[id]/restore/route.ts | 109 ++---- src/app/api/documents/inbound/bulk/route.ts | 44 +-- .../api/documents/inbound/deleted/route.ts | 92 +++++ src/app/api/documents/inbound/usage/route.ts | 11 + src/app/documents/deleted/page.tsx | 9 + .../documents/detail/use-document-detail.ts | 4 + .../documents/document-detail-sheet.tsx | 6 +- src/components/documents/documents-view.tsx | 6 +- .../__tests__/recently-deleted-view.test.tsx | 186 +++++++++ .../trash/__tests__/trash-utils.test.ts | 41 ++ .../documents/trash/recently-deleted-view.tsx | 270 +++++++++++++ src/components/documents/trash/trash-utils.ts | 31 ++ .../documents/trash/use-recently-deleted.ts | 115 ++++++ .../documents/vault/use-document-usage.ts | 21 + .../documents/vault/use-vault-bulk.ts | 8 +- .../documents/vault/use-vault-library.ts | 14 +- .../documents/vault/vault-page-actions.tsx | 36 +- src/lib/documents/inbound-constants.ts | 3 + src/lib/documents/tombstones.ts | 226 +++++++++++ src/lib/documents/vault-read-dtos.ts | 94 +++++ src/lib/jobs/document-purge-queue.ts | 17 + src/lib/jobs/document-purge.ts | 77 +--- src/lib/jobs/registry/maintenance.ts | 2 +- src/lib/jobs/soft-delete-retention.ts | 2 +- src/lib/openapi/error-codes.ts | 1 + src/lib/openapi/routes/documents.ts | 106 +++++- src/lib/query-keys/documents.ts | 6 + src/lib/validations/inbound-documents.ts | 46 +-- .../documents-recently-deleted.test.ts | 359 ++++++++++++++++++ 44 files changed, 2277 insertions(+), 270 deletions(-) create mode 100644 src/app/api/documents/inbound/[id]/purge/route.ts create mode 100644 src/app/api/documents/inbound/deleted/route.ts create mode 100644 src/app/documents/deleted/page.tsx create mode 100644 src/components/documents/trash/__tests__/recently-deleted-view.test.tsx create mode 100644 src/components/documents/trash/__tests__/trash-utils.test.ts create mode 100644 src/components/documents/trash/recently-deleted-view.tsx create mode 100644 src/components/documents/trash/trash-utils.ts create mode 100644 src/components/documents/trash/use-recently-deleted.ts create mode 100644 src/components/documents/vault/use-document-usage.ts create mode 100644 src/lib/documents/tombstones.ts create mode 100644 src/lib/documents/vault-read-dtos.ts create mode 100644 src/lib/jobs/document-purge-queue.ts create mode 100644 tests/integration/documents-recently-deleted.test.ts diff --git a/docs/api/openapi.yaml b/docs/api/openapi.yaml index 3fa11ca47..cf89cde46 100644 --- a/docs/api/openapi.yaml +++ b/docs/api/openapi.yaml @@ -19558,9 +19558,10 @@ paths: tags: - Documents summary: Restore a discarded document - description: Clears the soft-delete tombstone within the 30-day undo grace. 409 when the tombstone is already purged - (the undo window is over) or when an identical live document blocks the restore (`meta.reason = - "duplicateExists"`). Restoring an already-live document is a no-op success. + description: Clears the soft-delete tombstone within the 30-day undo grace, and forgets any import-ledger entry for the + document's source keys. 409 when the tombstone is already purged (the undo window is over, or the document was + deleted for good) or when an identical live document blocks the restore (`meta.reason = "duplicateExists"`). + Restoring an already-live document is a no-op success. parameters: - name: id in: path @@ -19584,6 +19585,60 @@ paths: $ref: "#/components/schemas/ErrorEnvelope" "422": *a4 "429": *a5 + /api/documents/inbound/deleted: + get: + tags: + - Documents + summary: List Recently deleted + description: "The caller's deleted documents that the purge has not removed yet, newest deletion first, metadata only. + Each carries `purgeAt`, when the daily purge becomes due for it. Restore one with `POST + /api/documents/inbound/{id}/restore` (or the bulk `restore` action); remove one for good ahead of time with + `POST /api/documents/inbound/{id}/purge`. Capped at 200 rows (`meta.truncated`, `meta.limit`). Owner-only: + refused while acting inside somebody else's record." + responses: + "200": + description: Recently deleted. + content: + application/json: + schema: + $ref: "#/components/schemas/DeletedDocumentListEnvelope" + "401": *a1 + "403": *a9 + "422": *a4 + "429": *a5 + /api/documents/inbound/{id}/purge: + post: + tags: + - Documents + summary: Delete a deleted document for good + description: "Removes one deleted document now instead of when the grace window ends, through the same purge the daily + job runs: the row and everything that cascades from it go, and an imported document's source keys are kept in + the import ledger, so an importer re-sending it is still answered as deleted. Only a deleted document can be + purged: a live one answers 409 `documents.inbound.notDeleted`. 404 `documents.inbound.notFound` when the caller + has no such document. Owner-only." + parameters: + - name: id + in: path + required: true + schema: + type: string + responses: + "200": + description: Removed for good. + content: + application/json: + schema: + $ref: "#/components/schemas/DocumentPurgeResultEnvelope" + "401": *a1 + "403": *a9 + "409": + description: The document is not deleted. + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorEnvelope" + "422": *a4 + "429": *a5 /api/documents/inbound/{id}/original: get: tags: @@ -38386,14 +38441,15 @@ components: documents.inbound.episodeNotFound, documents.inbound.extractFailed, documents.inbound.factNotFound, documents.inbound.factNotPending, documents.inbound.factTypeMismatch, documents.inbound.fileTooLarge, documents.inbound.fileType, documents.inbound.invalidMetadata, documents.inbound.invalidQuery, - documents.inbound.invalidUpdate, documents.inbound.localOcrDisabled, documents.inbound.notFound, - documents.inbound.notIndexed, documents.inbound.outputLimit, documents.inbound.pdfNeedsAnthropic, - documents.inbound.providerUnsupported, documents.inbound.quotaExceeded, documents.inbound.rateLimited, - documents.inbound.restoreDuplicate, documents.inbound.restoreGone, documents.inbound.sourceAliasLimit, - documents.inbound.uploadBusy, documents.inbound.uploadTimeout, documents.inbound.vaccinationNotFound, - documents.sources.authRefused, documents.sources.badResponse, documents.sources.browserOnly, - documents.sources.invalidAddress, documents.sources.linkTargetNotFound, documents.sources.notConnected, - documents.sources.notFound, documents.sources.organizationRequired, documents.sources.originNotAllowed, + documents.inbound.invalidUpdate, documents.inbound.localOcrDisabled, documents.inbound.notDeleted, + documents.inbound.notFound, documents.inbound.notIndexed, documents.inbound.outputLimit, + documents.inbound.pdfNeedsAnthropic, documents.inbound.providerUnsupported, + documents.inbound.quotaExceeded, documents.inbound.rateLimited, documents.inbound.restoreDuplicate, + documents.inbound.restoreGone, documents.inbound.sourceAliasLimit, documents.inbound.uploadBusy, + documents.inbound.uploadTimeout, documents.inbound.vaccinationNotFound, documents.sources.authRefused, + documents.sources.badResponse, documents.sources.browserOnly, documents.sources.invalidAddress, + documents.sources.linkTargetNotFound, documents.sources.notConnected, documents.sources.notFound, + documents.sources.organizationRequired, documents.sources.originNotAllowed, documents.sources.permissionMissing, documents.sources.rateLimited, documents.sources.redirected, documents.sources.tokenRequired, documents.sources.unavailable, documents.sources.unreachable, documents.sources.versionTooOld; encounter — encounter.invalid, encounter.link.invalid, @@ -79845,6 +79901,13 @@ components: - indexedCount - totalCount additionalProperties: false + deletedCount: + type: integer + minimum: -9007199254740991 + maximum: 9007199254740991 + description: How many of the caller's documents are deleted and not yet purged (Recently deleted, `GET + /api/documents/inbound/deleted`). Always 0 while acting inside somebody else's record, where Recently + deleted is not offered. required: - usedBytes - quotaBytes @@ -79854,6 +79917,7 @@ components: - linkedProcedures - assistAvailable - contentIndex + - deletedCount additionalProperties: false DocumentsLayoutEnvelope: type: object @@ -80369,6 +80433,133 @@ components: - data - error additionalProperties: false + DeletedDocumentListEnvelope: + type: object + properties: + data: + $ref: "#/components/schemas/DeletedDocumentList" + error: + type: "null" + meta: + $ref: "#/components/schemas/ListCapMeta" + required: + - data + - error + - meta + additionalProperties: false + DeletedDocumentList: + type: object + properties: + documents: + type: array + items: + $ref: "#/components/schemas/DeletedDocument" + graceDays: + type: integer + minimum: -9007199254740991 + maximum: 9007199254740991 + description: Days a deleted document stays restorable. + required: + - documents + - graceDays + additionalProperties: false + DeletedDocument: + type: object + properties: + id: + type: string + kind: + type: string + enum: + - DOCTOR_REPORT + - DISCHARGE_LETTER + - LAB_RESULT + - IMAGING + - PRESCRIPTION + - REFERRAL + - INSURANCE + - VACCINATION + - SICK_NOTE + - OTHER + title: + type: + - string + - "null" + filename: + type: + - string + - "null" + mimeType: + type: string + byteSize: + type: integer + minimum: -9007199254740991 + maximum: 9007199254740991 + documentDate: + description: User-set filing date (YYYY-MM-DD), or null. + type: + - string + - "null" + sourceSystem: + anyOf: + - type: string + enum: + - PAPERLESS + - PAPRA + - OTHER + - type: "null" + description: The system an imported document came from, or null for one added in HealthLog. + deletedAt: + type: string + format: date-time + pattern: ^(?:(?:\d\d[2468][048]|\d\d[13579][26]|\d\d0[48]|[02468][048]00|[13579][26]00)-02-29|\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\d|30)|(?:02)-(?:0[1-9]|1\d|2[0-8])))T(?:(?:[01]\d|2[0-3]):[0-5]\d:[0-5]\d(?:\.\d+)?(?:Z|([+-](?:[01]\d|2[0-3]):[0-5]\d)))$ + purgeAt: + type: string + format: date-time + pattern: ^(?:(?:\d\d[2468][048]|\d\d[13579][26]|\d\d0[48]|[02468][048]00|[13579][26]00)-02-29|\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\d|30)|(?:02)-(?:0[1-9]|1\d|2[0-8])))T(?:(?:[01]\d|2[0-3]):[0-5]\d:[0-5]\d(?:\.\d+)?(?:Z|([+-](?:[01]\d|2[0-3]):[0-5]\d)))$ + description: "When the daily purge becomes due for this document: `deletedAt` plus `graceDays`. The purge runs once a + day, so the document is removed at the first run after this instant." + required: + - id + - kind + - title + - filename + - mimeType + - byteSize + - documentDate + - sourceSystem + - deletedAt + - purgeAt + additionalProperties: false + DocumentPurgeResultEnvelope: + type: object + properties: + data: + $ref: "#/components/schemas/DocumentPurgeResult" + error: + type: "null" + meta: + type: object + properties: + requestId: + type: string + additionalProperties: false + required: + - data + - error + additionalProperties: false + DocumentPurgeResult: + type: object + properties: + id: + type: string + purged: + type: boolean + const: true + required: + - id + - purged + additionalProperties: false InboundExtractEnvelope: type: object properties: diff --git a/messages/de.json b/messages/de.json index 8ad355065..4ef2f6ce8 100644 --- a/messages/de.json +++ b/messages/de.json @@ -11651,7 +11651,8 @@ "toast": { "duplicate": "Bereits gespeichert. Das vorhandene Dokument ist hervorgehoben.", "deleted": "Dokument gelöscht", - "restoreFailed": "Das Dokument konnte nicht wiederhergestellt werden." + "restoreFailed": "Das Dokument konnte nicht wiederhergestellt werden.", + "deletedHint": "Gelöschte Dokumente bleiben {days} Tage unter Zuletzt gelöscht." }, "timeline": { "loadingMore": "Mehr wird geladen…", @@ -11946,8 +11947,32 @@ "deleteConfirm": { "titleOne": "Dieses Dokument löschen?", "titleMany": "{count} Dokumente löschen?", - "body": "Direkt danach lässt sich das noch rückgängig machen. Nach 30 Tagen wird es endgültig entfernt.", + "body": "Gelöschte Dokumente bleiben {days} Tage unter Zuletzt gelöscht, danach werden sie endgültig entfernt.", "action": "Löschen" + }, + "trash": { + "title": "Zuletzt gelöscht", + "subtitle": "Gelöschte Dokumente bleiben {days} Tage hier, danach werden sie endgültig entfernt.", + "back": "Dokumente", + "entryLabel": "Zuletzt gelöscht ({count})", + "loadError": "Zuletzt gelöscht konnte nicht geladen werden.", + "emptyTitle": "Nichts unter Zuletzt gelöscht", + "emptyDescription": "Was du löschst, wartet hier {days} Tage, bevor es endgültig entfernt wird.", + "count": "Gelöschte Dokumente ({count})", + "restoreAll": "Alle wiederherstellen", + "restore": "Wiederherstellen", + "purge": "Jetzt löschen", + "deletedOn": "Gelöscht am {date}.", + "removesIn": "Wird in {days} Tagen endgültig entfernt.", + "removesSoon": "Wird innerhalb eines Tages endgültig entfernt.", + "restored": "Dokument wiederhergestellt", + "openRestored": "Öffnen", + "restoreDuplicate": "Ein identisches Dokument ist schon gespeichert, deshalb kann diese Kopie nicht wiederhergestellt werden.", + "restorePartial": "{failed} von {total} konnten nicht wiederhergestellt werden", + "purged": "Dokument endgültig gelöscht", + "purgeFailed": "Das Dokument konnte nicht gelöscht werden.", + "purgeConfirmTitle": "Dieses Dokument endgültig löschen?", + "purgeConfirmBody": "Es wird jetzt entfernt statt am Ende der Wartezeit und lässt sich nicht wiederherstellen." } }, "selfDescription": { diff --git a/messages/en.json b/messages/en.json index d61063798..71c85501f 100644 --- a/messages/en.json +++ b/messages/en.json @@ -11651,7 +11651,8 @@ "toast": { "duplicate": "Already stored. The existing document is highlighted.", "deleted": "Document deleted", - "restoreFailed": "Couldn't restore the document." + "restoreFailed": "Couldn't restore the document.", + "deletedHint": "Deleted documents stay in Recently deleted for {days} days." }, "timeline": { "loadingMore": "Loading more…", @@ -11946,8 +11947,32 @@ "deleteConfirm": { "titleOne": "Delete this document?", "titleMany": "Delete {count} documents?", - "body": "You can undo this right afterwards. After 30 days it is removed for good.", + "body": "Deleted documents stay in Recently deleted for {days} days, then they are removed for good.", "action": "Delete" + }, + "trash": { + "title": "Recently deleted", + "subtitle": "Deleted documents stay here for {days} days, then they are removed for good.", + "back": "Documents", + "entryLabel": "Recently deleted ({count})", + "loadError": "Couldn't load Recently deleted.", + "emptyTitle": "Nothing in Recently deleted", + "emptyDescription": "Documents you delete wait here for {days} days before they are removed for good.", + "count": "Deleted documents ({count})", + "restoreAll": "Restore all", + "restore": "Restore", + "purge": "Delete now", + "deletedOn": "Deleted {date}.", + "removesIn": "Removed for good in {days} days.", + "removesSoon": "Removed for good within a day.", + "restored": "Document restored", + "openRestored": "Open", + "restoreDuplicate": "An identical document is already stored, so this copy cannot be restored.", + "restorePartial": "{failed} of {total} could not be restored", + "purged": "Document deleted for good", + "purgeFailed": "Couldn't delete the document.", + "purgeConfirmTitle": "Delete this document for good?", + "purgeConfirmBody": "It is removed now instead of at the end of the waiting period, and cannot be restored." } }, "selfDescription": { diff --git a/messages/es.json b/messages/es.json index 39f65500d..a90623fc8 100644 --- a/messages/es.json +++ b/messages/es.json @@ -11651,7 +11651,8 @@ "toast": { "duplicate": "Ya estaba guardado. Se resalta el documento existente.", "deleted": "Documento eliminado", - "restoreFailed": "No se pudo restaurar el documento." + "restoreFailed": "No se pudo restaurar el documento.", + "deletedHint": "Los documentos eliminados se quedan en Eliminados recientemente durante {days} días." }, "timeline": { "loadingMore": "Cargando más…", @@ -11946,8 +11947,32 @@ "deleteConfirm": { "titleOne": "¿Eliminar este documento?", "titleMany": "¿Eliminar {count} documentos?", - "body": "Puedes deshacerlo justo después. A los 30 días se elimina definitivamente.", + "body": "Los documentos eliminados se quedan en Eliminados recientemente durante {days} días; después se eliminan definitivamente.", "action": "Eliminar" + }, + "trash": { + "title": "Eliminados recientemente", + "subtitle": "Los documentos eliminados se quedan aquí {days} días; después se eliminan definitivamente.", + "back": "Documentos", + "entryLabel": "Eliminados recientemente ({count})", + "loadError": "No se pudo cargar Eliminados recientemente.", + "emptyTitle": "No hay nada en Eliminados recientemente", + "emptyDescription": "Lo que eliminas espera aquí {days} días antes de eliminarse definitivamente.", + "count": "Documentos eliminados ({count})", + "restoreAll": "Restaurar todo", + "restore": "Restaurar", + "purge": "Eliminar ahora", + "deletedOn": "Eliminado el {date}.", + "removesIn": "Se elimina definitivamente en {days} días.", + "removesSoon": "Se elimina definitivamente en menos de un día.", + "restored": "Documento restaurado", + "openRestored": "Abrir", + "restoreDuplicate": "Ya hay un documento idéntico guardado, así que esta copia no se puede restaurar.", + "restorePartial": "No se pudieron restaurar {failed} de {total}", + "purged": "Documento eliminado definitivamente", + "purgeFailed": "No se pudo eliminar el documento.", + "purgeConfirmTitle": "¿Eliminar este documento definitivamente?", + "purgeConfirmBody": "Se elimina ahora en lugar de al final del periodo de espera y no se puede restaurar." } }, "selfDescription": { diff --git a/messages/fr.json b/messages/fr.json index d484db95d..7d93dc773 100644 --- a/messages/fr.json +++ b/messages/fr.json @@ -11651,7 +11651,8 @@ "toast": { "duplicate": "Déjà enregistré. Le document existant est mis en évidence.", "deleted": "Document supprimé", - "restoreFailed": "Impossible de restaurer le document." + "restoreFailed": "Impossible de restaurer le document.", + "deletedHint": "Les documents supprimés restent {days} jours dans Supprimés récemment." }, "timeline": { "loadingMore": "Chargement…", @@ -11946,8 +11947,32 @@ "deleteConfirm": { "titleOne": "Supprimer ce document ?", "titleMany": "Supprimer {count} documents ?", - "body": "Vous pouvez annuler juste après. Au bout de 30 jours, la suppression est définitive.", + "body": "Les documents supprimés restent {days} jours dans Supprimés récemment, puis ils sont supprimés définitivement.", "action": "Supprimer" + }, + "trash": { + "title": "Supprimés récemment", + "subtitle": "Les documents supprimés restent ici {days} jours, puis ils sont supprimés définitivement.", + "back": "Documents", + "entryLabel": "Supprimés récemment ({count})", + "loadError": "Impossible de charger Supprimés récemment.", + "emptyTitle": "Rien dans Supprimés récemment", + "emptyDescription": "Ce que vous supprimez attend ici {days} jours avant d'être supprimé définitivement.", + "count": "Documents supprimés ({count})", + "restoreAll": "Tout restaurer", + "restore": "Restaurer", + "purge": "Supprimer maintenant", + "deletedOn": "Supprimé le {date}.", + "removesIn": "Supprimé définitivement dans {days} jours.", + "removesSoon": "Supprimé définitivement dans moins d'un jour.", + "restored": "Document restauré", + "openRestored": "Ouvrir", + "restoreDuplicate": "Un document identique est déjà enregistré, cette copie ne peut donc pas être restaurée.", + "restorePartial": "{failed} sur {total} n'ont pas pu être restaurés", + "purged": "Document supprimé définitivement", + "purgeFailed": "Impossible de supprimer le document.", + "purgeConfirmTitle": "Supprimer ce document définitivement ?", + "purgeConfirmBody": "Il est supprimé maintenant plutôt qu'à la fin du délai d'attente et ne peut pas être restauré." } }, "selfDescription": { diff --git a/messages/it.json b/messages/it.json index 4d929eec9..1e6f6cce6 100644 --- a/messages/it.json +++ b/messages/it.json @@ -11651,7 +11651,8 @@ "toast": { "duplicate": "Già archiviato. Il documento esistente è evidenziato.", "deleted": "Documento eliminato", - "restoreFailed": "Impossibile ripristinare il documento." + "restoreFailed": "Impossibile ripristinare il documento.", + "deletedHint": "I documenti eliminati restano in Eliminati di recente per {days} giorni." }, "timeline": { "loadingMore": "Caricamento…", @@ -11946,8 +11947,32 @@ "deleteConfirm": { "titleOne": "Eliminare questo documento?", "titleMany": "Eliminare {count} documenti?", - "body": "Puoi annullare subito dopo. Dopo 30 giorni viene rimosso definitivamente.", + "body": "I documenti eliminati restano in Eliminati di recente per {days} giorni, poi vengono rimossi definitivamente.", "action": "Elimina" + }, + "trash": { + "title": "Eliminati di recente", + "subtitle": "I documenti eliminati restano qui per {days} giorni, poi vengono rimossi definitivamente.", + "back": "Documenti", + "entryLabel": "Eliminati di recente ({count})", + "loadError": "Impossibile caricare Eliminati di recente.", + "emptyTitle": "Niente in Eliminati di recente", + "emptyDescription": "Ciò che elimini resta qui per {days} giorni prima di essere rimosso definitivamente.", + "count": "Documenti eliminati ({count})", + "restoreAll": "Ripristina tutto", + "restore": "Ripristina", + "purge": "Elimina ora", + "deletedOn": "Eliminato il {date}.", + "removesIn": "Rimosso definitivamente tra {days} giorni.", + "removesSoon": "Rimosso definitivamente entro un giorno.", + "restored": "Documento ripristinato", + "openRestored": "Apri", + "restoreDuplicate": "Un documento identico è già salvato, quindi questa copia non può essere ripristinata.", + "restorePartial": "{failed} su {total} non sono stati ripristinati", + "purged": "Documento eliminato definitivamente", + "purgeFailed": "Impossibile eliminare il documento.", + "purgeConfirmTitle": "Eliminare definitivamente questo documento?", + "purgeConfirmBody": "Viene rimosso ora anziché alla fine del periodo di attesa e non può essere ripristinato." } }, "selfDescription": { diff --git a/messages/ko.json b/messages/ko.json index c1ed15b59..dc0f9212b 100644 --- a/messages/ko.json +++ b/messages/ko.json @@ -11651,7 +11651,8 @@ "toast": { "duplicate": "이미 저장돼 있어요. 기존 문서를 표시해요.", "deleted": "문서를 삭제했어요", - "restoreFailed": "문서를 복구하지 못했어요." + "restoreFailed": "문서를 복구하지 못했어요.", + "deletedHint": "삭제한 문서는 {days}일 동안 최근 삭제됨에 남아 있어요." }, "timeline": { "loadingMore": "더 불러오는 중…", @@ -11946,8 +11947,32 @@ "deleteConfirm": { "titleOne": "이 문서를 삭제할까요?", "titleMany": "문서 {count}개를 삭제할까요?", - "body": "삭제 직후에는 되돌릴 수 있습니다. 30일이 지나면 완전히 삭제됩니다.", + "body": "삭제한 문서는 {days}일 동안 최근 삭제됨에 남아 있다가 완전히 삭제됩니다.", "action": "삭제" + }, + "trash": { + "title": "최근 삭제됨", + "subtitle": "삭제한 문서는 {days}일 동안 여기 남아 있다가 완전히 삭제됩니다.", + "back": "문서", + "entryLabel": "최근 삭제됨 ({count})", + "loadError": "최근 삭제됨을 불러오지 못했어요.", + "emptyTitle": "최근 삭제됨이 비어 있어요", + "emptyDescription": "삭제한 문서는 완전히 삭제되기 전까지 {days}일 동안 여기서 기다려요.", + "count": "삭제한 문서 ({count})", + "restoreAll": "모두 복구", + "restore": "복구", + "purge": "지금 삭제", + "deletedOn": "{date}에 삭제함.", + "removesIn": "{days}일 후 완전히 삭제됩니다.", + "removesSoon": "하루 안에 완전히 삭제됩니다.", + "restored": "문서를 복구했어요", + "openRestored": "열기", + "restoreDuplicate": "같은 문서가 이미 저장되어 있어서 이 사본은 복구할 수 없어요.", + "restorePartial": "{total}개 중 {failed}개를 복구하지 못했어요", + "purged": "문서를 완전히 삭제했어요", + "purgeFailed": "문서를 삭제하지 못했어요.", + "purgeConfirmTitle": "이 문서를 완전히 삭제할까요?", + "purgeConfirmBody": "대기 기간이 끝나기 전에 지금 삭제되며 복구할 수 없습니다." } }, "selfDescription": { diff --git a/messages/nl.json b/messages/nl.json index e353a06e6..a831c0207 100644 --- a/messages/nl.json +++ b/messages/nl.json @@ -11651,7 +11651,8 @@ "toast": { "duplicate": "Al opgeslagen. Het bestaande document is gemarkeerd.", "deleted": "Document verwijderd", - "restoreFailed": "Het document kon niet worden hersteld." + "restoreFailed": "Het document kon niet worden hersteld.", + "deletedHint": "Verwijderde documenten blijven {days} dagen in Recent verwijderd." }, "timeline": { "loadingMore": "Meer laden…", @@ -11946,8 +11947,32 @@ "deleteConfirm": { "titleOne": "Dit document verwijderen?", "titleMany": "{count} documenten verwijderen?", - "body": "Je kunt dit direct daarna ongedaan maken. Na 30 dagen wordt het definitief verwijderd.", + "body": "Verwijderde documenten blijven {days} dagen in Recent verwijderd en worden daarna definitief verwijderd.", "action": "Verwijderen" + }, + "trash": { + "title": "Recent verwijderd", + "subtitle": "Verwijderde documenten blijven hier {days} dagen en worden daarna definitief verwijderd.", + "back": "Documenten", + "entryLabel": "Recent verwijderd ({count})", + "loadError": "Recent verwijderd kon niet worden geladen.", + "emptyTitle": "Niets in Recent verwijderd", + "emptyDescription": "Wat je verwijdert, wacht hier {days} dagen voordat het definitief wordt verwijderd.", + "count": "Verwijderde documenten ({count})", + "restoreAll": "Alles herstellen", + "restore": "Herstellen", + "purge": "Nu verwijderen", + "deletedOn": "Verwijderd op {date}.", + "removesIn": "Wordt over {days} dagen definitief verwijderd.", + "removesSoon": "Wordt binnen een dag definitief verwijderd.", + "restored": "Document hersteld", + "openRestored": "Openen", + "restoreDuplicate": "Er is al een identiek document opgeslagen, dus deze kopie kan niet worden hersteld.", + "restorePartial": "{failed} van {total} konden niet worden hersteld", + "purged": "Document definitief verwijderd", + "purgeFailed": "Het document kon niet worden verwijderd.", + "purgeConfirmTitle": "Dit document definitief verwijderen?", + "purgeConfirmBody": "Het wordt nu verwijderd in plaats van aan het einde van de wachttijd en kan niet worden hersteld." } }, "selfDescription": { diff --git a/messages/pl.json b/messages/pl.json index f388f0a01..679c7ee7d 100644 --- a/messages/pl.json +++ b/messages/pl.json @@ -11651,7 +11651,8 @@ "toast": { "duplicate": "Już zapisano. Istniejący dokument został wyróżniony.", "deleted": "Dokument usunięty", - "restoreFailed": "Nie udało się przywrócić dokumentu." + "restoreFailed": "Nie udało się przywrócić dokumentu.", + "deletedHint": "Usunięte dokumenty zostają w sekcji Ostatnio usunięte przez {days} dni." }, "timeline": { "loadingMore": "Wczytywanie…", @@ -11946,8 +11947,32 @@ "deleteConfirm": { "titleOne": "Usunąć ten dokument?", "titleMany": "Usunąć dokumenty ({count})?", - "body": "Zaraz potem możesz to cofnąć. Po 30 dniach zostanie usunięte na stałe.", + "body": "Usunięte dokumenty zostają w sekcji Ostatnio usunięte przez {days} dni, a potem są usuwane na stałe.", "action": "Usuń" + }, + "trash": { + "title": "Ostatnio usunięte", + "subtitle": "Usunięte dokumenty zostają tu przez {days} dni, a potem są usuwane na stałe.", + "back": "Dokumenty", + "entryLabel": "Ostatnio usunięte ({count})", + "loadError": "Nie udało się wczytać sekcji Ostatnio usunięte.", + "emptyTitle": "Brak elementów w sekcji Ostatnio usunięte", + "emptyDescription": "To, co usuniesz, czeka tu {days} dni, zanim zostanie usunięte na stałe.", + "count": "Usunięte dokumenty ({count})", + "restoreAll": "Przywróć wszystkie", + "restore": "Przywróć", + "purge": "Usuń teraz", + "deletedOn": "Usunięto {date}.", + "removesIn": "Zostanie usunięty na stałe za {days} dni.", + "removesSoon": "Zostanie usunięty na stałe w ciągu doby.", + "restored": "Przywrócono dokument", + "openRestored": "Otwórz", + "restoreDuplicate": "Identyczny dokument jest już zapisany, więc tej kopii nie można przywrócić.", + "restorePartial": "Nie udało się przywrócić {failed} z {total}", + "purged": "Dokument usunięty na stałe", + "purgeFailed": "Nie udało się usunąć dokumentu.", + "purgeConfirmTitle": "Usunąć ten dokument na stałe?", + "purgeConfirmBody": "Zostanie usunięty teraz zamiast po okresie oczekiwania i nie będzie można go przywrócić." } }, "selfDescription": { diff --git a/messages/pt-BR.json b/messages/pt-BR.json index e0793939e..d19fdec09 100644 --- a/messages/pt-BR.json +++ b/messages/pt-BR.json @@ -11651,7 +11651,8 @@ "toast": { "duplicate": "Já armazenado. O documento existente está destacado.", "deleted": "Documento excluído", - "restoreFailed": "Não foi possível restaurar o documento." + "restoreFailed": "Não foi possível restaurar o documento.", + "deletedHint": "Documentos excluídos ficam em Excluídos recentemente por {days} dias." }, "timeline": { "loadingMore": "Carregando mais…", @@ -11946,8 +11947,32 @@ "deleteConfirm": { "titleOne": "Excluir este documento?", "titleMany": "Excluir {count} documentos?", - "body": "Você pode desfazer isso logo depois. Após 30 dias, é removido para sempre.", + "body": "Documentos excluídos ficam em Excluídos recentemente por {days} dias e depois são removidos para sempre.", "action": "Excluir" + }, + "trash": { + "title": "Excluídos recentemente", + "subtitle": "Documentos excluídos ficam aqui por {days} dias e depois são removidos para sempre.", + "back": "Documentos", + "entryLabel": "Excluídos recentemente ({count})", + "loadError": "Não foi possível carregar Excluídos recentemente.", + "emptyTitle": "Nada em Excluídos recentemente", + "emptyDescription": "O que você exclui espera aqui por {days} dias antes de ser removido para sempre.", + "count": "Documentos excluídos ({count})", + "restoreAll": "Restaurar todos", + "restore": "Restaurar", + "purge": "Excluir agora", + "deletedOn": "Excluído em {date}.", + "removesIn": "Removido para sempre em {days} dias.", + "removesSoon": "Removido para sempre em menos de um dia.", + "restored": "Documento restaurado", + "openRestored": "Abrir", + "restoreDuplicate": "Já existe um documento idêntico salvo, então esta cópia não pode ser restaurada.", + "restorePartial": "{failed} de {total} não puderam ser restaurados", + "purged": "Documento excluído para sempre", + "purgeFailed": "Não foi possível excluir o documento.", + "purgeConfirmTitle": "Excluir este documento para sempre?", + "purgeConfirmBody": "Ele é removido agora, e não no fim do período de espera, e não pode ser restaurado." } }, "selfDescription": { diff --git a/src/__tests__/baselines/source-file-lines.json b/src/__tests__/baselines/source-file-lines.json index 7478dfb91..3721a2163 100644 --- a/src/__tests__/baselines/source-file-lines.json +++ b/src/__tests__/baselines/source-file-lines.json @@ -1,7 +1,7 @@ { "$comment": "Production files over 800 lines and their length. Maintained by src/__tests__/source-size-ratchet.test.ts.", "count": 88, - "total": 98310, + "total": 98280, "entries": { "app/medications/page-client.tsx": 839, "components/charts/mood-chart.tsx": 1166, @@ -87,7 +87,7 @@ "lib/sharing/grants.ts": 807, "lib/support/report.ts": 903, "lib/telegram-webhook-handlers.ts": 1341, - "lib/validations/inbound-documents.ts": 1048, + "lib/validations/inbound-documents.ts": 1018, "lib/withings/sync-sleep.ts": 884, "lib/withings/sync.ts": 933, "proxy.ts": 870 diff --git a/src/__tests__/baselines/source-function-lines.json b/src/__tests__/baselines/source-function-lines.json index c97a01137..7b0728781 100644 --- a/src/__tests__/baselines/source-function-lines.json +++ b/src/__tests__/baselines/source-function-lines.json @@ -1,7 +1,7 @@ { "$comment": "Production functions over 200 lines, keyed \"file :: name\", and their length. Maintained by src/__tests__/source-size-ratchet.test.ts.", - "count": 352, - "total": 113949, + "count": 351, + "total": 113742, "entries": { "app/achievements/page.tsx :: AchievementsPage": 206, "app/api/admin/backups/upload/route.ts :: apiHandler(arg)": 314, @@ -16,7 +16,6 @@ "app/api/cycle/day-logs/bulk/route.ts :: postBulk": 213, "app/api/cycle/insights/route.ts :: apiHandler(arg)": 226, "app/api/devices/route.ts :: apiHandler(arg)": 226, - "app/api/documents/inbound/bulk/route.ts :: apiHandler(arg)": 207, "app/api/encounters/[id]/route.ts :: apiHandler(arg) #2": 269, "app/api/environment/route.ts :: apiHandler(arg)": 201, "app/api/export/health-record/route.ts :: apiHandler(arg)": 241, diff --git a/src/__tests__/destructive-control-guard.test.ts b/src/__tests__/destructive-control-guard.test.ts index dcbff7a92..7d647f30f 100644 --- a/src/__tests__/destructive-control-guard.test.ts +++ b/src/__tests__/destructive-control-guard.test.ts @@ -627,10 +627,18 @@ const REGISTRY: DestructiveEntry[] = [ { control: "bulk delete, and the Delete/Backspace key on a focused card", reason: - "Same undo-not-confirmation choice as the detail sheet, and the bulk path raises one aggregate undo for the whole batch. The keyboard path is the weakest link in that argument, since a keystroke needs no aim and the undo lives only in a toast with no trash view behind it; it is recorded here rather than silently accepted, and closing it means giving documents a recovery surface, not a dialog.", + "Same undo-not-confirmation choice as the detail sheet, and the bulk path raises one aggregate undo for the whole batch. The keyboard path was the weakest link in that argument while the undo lived only in a toast; Recently deleted (/documents/deleted) is now the recovery surface behind it, where every deleted document can be restored for 30 days.", }, ], }, + { + file: "components/documents/trash/use-recently-deleted.ts", + destroys: + "one already-deleted document, for good, ahead of the 30-day purge (Delete now in Recently deleted)", + triggers: ["components/documents/trash/recently-deleted-view.tsx"], + recovery: "permanent", + confirm: ["ConfirmDialog"], + }, // ── Account sharing ───────────────────────────────────────────────────── { diff --git a/src/__tests__/success-affordance-guard.test.ts b/src/__tests__/success-affordance-guard.test.ts index 2e1ec3bed..78eea0bbb 100644 --- a/src/__tests__/success-affordance-guard.test.ts +++ b/src/__tests__/success-affordance-guard.test.ts @@ -192,6 +192,13 @@ const PINNED_AFFORDANCES: Record< "src/components/documents/vault/use-vault-bulk.ts": { "toast.success": 2, }, + // Recently deleted: the one restore's toast, which carries an "Open" + // action the shared outcome toast has no slot for. It fires only on the + // restore route's 200 for that one document, i.e. a document back in the + // vault. Restore-all and delete-now go through toastWrittenOutcome. + "src/components/documents/trash/use-recently-deleted.ts": { + "toast.success": 1, + }, // The document picker's per-row result: tinted only for the `imported` / // `duplicate` outcome the import route returned for that one document, i.e. // a document the vault now holds. The run's headline goes through diff --git a/src/app/api/documents/inbound/[id]/purge/route.ts b/src/app/api/documents/inbound/[id]/purge/route.ts new file mode 100644 index 000000000..303c8973c --- /dev/null +++ b/src/app/api/documents/inbound/[id]/purge/route.ts @@ -0,0 +1,105 @@ +/** + * Document vault: remove one deleted document for good, now. + * + * "Delete now" in Recently deleted. The daily job purges a deleted document + * once the grace window has passed; this does the same for one document + * ahead of time, through the same function (`purgeDocumentTombstones` in + * `src/lib/documents/tombstones.ts`): the row and everything that cascades + * from it go, and an imported document's source keys are carried into the + * import ledger in the same transaction, so a re-run of the importer does not + * bring it back. + * + * Only a deleted document can be purged: a live one answers 409 + * `documents.inbound.notDeleted`, so no single request takes a document from + * the vault straight out of the database. Owner-only, like delete and restore. + */ +import { NextRequest } from "next/server"; + +import { apiHandler, requireAuth } from "@/lib/api-handler"; +import { apiError, apiSuccess, getClientIp } from "@/lib/api-response"; +import { auditLog } from "@/lib/auth/audit"; +import { prisma } from "@/lib/db"; +import { purgeDocumentTombstoneNow } from "@/lib/documents/tombstones"; +import { annotate } from "@/lib/logging/context"; +import { requireModuleEnabled } from "@/lib/modules/gate"; +import { checkRateLimit, rateLimitHeaders } from "@/lib/rate-limit"; +import type { DocumentPurgeDto } from "@/lib/validations/inbound-documents"; + +type RouteParams = { params: Promise<{ id: string }> }; + +export const dynamic = "force-dynamic"; + +/** Shares the write bucket of the other owner-scoped document mutations. */ +const WRITE_LIMIT_PER_HOUR = 240; +const WRITE_WINDOW_MS = 60 * 60 * 1000; + +export const POST = apiHandler( + async (request: NextRequest, { params }: RouteParams) => { + const { user } = await requireAuth(); + const gate = await requireModuleEnabled(user.id, "inboundDocuments"); + if (!gate.enabled) return gate.response; + + const rl = await checkRateLimit( + `documents-write:${user.id}`, + WRITE_LIMIT_PER_HOUR, + WRITE_WINDOW_MS, + ); + if (!rl.allowed) { + const response = apiError("Too many requests. Try again later.", 429, { + errorCode: "documents.inbound.rateLimited", + }); + for (const [k, v] of Object.entries(rateLimitHeaders(rl))) { + response.headers.set(k, v); + } + return response; + } + + const { id } = await params; + const row = await prisma.inboundDocument.findFirst({ + where: { id, userId: user.id }, + select: { id: true, deletedAt: true }, + }); + if (!row) { + return apiError("Document not found", 404, { + errorCode: "documents.inbound.notFound", + }); + } + if (row.deletedAt === null) { + return apiError("Only a deleted document can be removed for good.", 409, { + errorCode: "documents.inbound.notDeleted", + }); + } + + const purged = await purgeDocumentTombstoneNow(user.id, row.id); + if (!purged) { + // Lost a race between the read above and the purge: a restore brought + // it back (409, as above), or the daily purge took it first, which is + // what was asked for. + const now = await prisma.inboundDocument.findFirst({ + where: { id: row.id, userId: user.id }, + select: { deletedAt: true }, + }); + if (now && now.deletedAt === null) { + return apiError( + "Only a deleted document can be removed for good.", + 409, + { errorCode: "documents.inbound.notDeleted" }, + ); + } + } + + await auditLog("documents.inbound.purge", { + userId: user.id, + ipAddress: getClientIp(request), + details: { documentId: row.id }, + }); + + annotate({ + action: { name: "documents.trash.purge" }, + meta: { documentId: row.id, raced: !purged }, + }); + + const body: DocumentPurgeDto = { id: row.id, purged: true }; + return apiSuccess(body); + }, +); diff --git a/src/app/api/documents/inbound/[id]/restore/route.ts b/src/app/api/documents/inbound/[id]/restore/route.ts index 68f847f48..9c556ceee 100644 --- a/src/app/api/documents/inbound/[id]/restore/route.ts +++ b/src/app/api/documents/inbound/[id]/restore/route.ts @@ -8,6 +8,8 @@ * indistinguishable from never-existed by construction, and owner-scoped * either way) or when restoring would collide with a live duplicate of the * same bytes (the `inbound_documents_user_sha_live` partial unique index). + * The restore itself is the shared one (`src/lib/documents/tombstones.ts`): + * it also clears any import-ledger entry for the document's source keys. */ import { NextRequest } from "next/server"; @@ -20,11 +22,10 @@ import { loadDocumentEncounterLinks, } from "@/lib/documents/links"; import { serialiseDocumentDetail } from "@/lib/documents/store"; +import { restoreDocumentTombstone } from "@/lib/documents/tombstones"; import { annotate } from "@/lib/logging/context"; import { requireModuleEnabled } from "@/lib/modules/gate"; -import { isP2002 } from "@/lib/prisma-errors"; import { checkRateLimit, rateLimitHeaders } from "@/lib/rate-limit"; -import type { InboundDocumentStatus } from "@/generated/prisma/client"; type RouteParams = { params: Promise<{ id: string }> }; @@ -34,18 +35,6 @@ export const dynamic = "force-dynamic"; const WRITE_LIMIT_PER_HOUR = 240; const WRITE_WINDOW_MS = 60 * 60 * 1000; -/** - * Post-restore lifecycle status, derived from the staged facts: no facts → - * STORED, any PENDING → EXTRACTED, otherwise every fact was acted on → - * CONFIRMED. (DELETE stamped DISCARDED; the prior status is not persisted.) - */ -function restoredStatus( - facts: Array<{ status: string }>, -): InboundDocumentStatus { - if (facts.length === 0) return "STORED"; - return facts.some((f) => f.status === "PENDING") ? "EXTRACTED" : "CONFIRMED"; -} - export const POST = apiHandler( async (request: NextRequest, { params }: RouteParams) => { const { user } = await requireAuth(); @@ -68,80 +57,26 @@ export const POST = apiHandler( } const { id } = await params; - const row = await prisma.inboundDocument.findFirst({ - where: { id, userId: user.id }, - omit: { contentEncrypted: true }, - include: { facts: { orderBy: { createdAt: "asc" } } }, - }); - - if (!row) { - // Purged (past the 30-day grace) or never existed — the tombstone is - // gone either way, so the undo window is over: 409, not 404. + const outcome = await restoreDocumentTombstone(user.id, id); + if (outcome.kind === "gone") { + // Purged (past the 30-day grace, or deleted now from Recently deleted) + // or never existed — the tombstone is gone either way, so the undo + // window is over: 409, not 404. return apiError("This document can no longer be restored.", 409, { errorCode: "documents.inbound.restoreGone", reason: "purged", }); } - - if (row.deletedAt === null) { - // Already live — restoring twice is a no-op, not an error (the undo - // toast may fire after a concurrent restore). - const [links, visitLinks] = await Promise.all([ - loadConditionLinks(user.id, [row.id]), - loadDocumentEncounterLinks(user.id, [row.id]), - ]); - return apiSuccess( - serialiseDocumentDetail( - row, - row.facts, - links.get(row.id) ?? [], - false, - null, - false, - visitLinks.get(row.id) ?? [], - ), - ); - } - - // A live duplicate of the same bytes blocks the restore (the partial - // unique index would reject it; check first for a clean 409). - if (row.contentSha256) { - const conflict = await prisma.inboundDocument.findFirst({ - where: { - userId: user.id, - contentSha256: row.contentSha256, - deletedAt: null, - id: { not: row.id }, - }, - select: { id: true }, + if (outcome.kind === "duplicate") { + return apiError("An identical document is already stored.", 409, { + errorCode: "documents.inbound.restoreDuplicate", + reason: "duplicateExists", + ...(outcome.existingId ? { existingId: outcome.existingId } : {}), }); - if (conflict) { - return apiError("An identical document is already stored.", 409, { - errorCode: "documents.inbound.restoreDuplicate", - reason: "duplicateExists", - existingId: conflict.id, - }); - } - } - - try { - await prisma.inboundDocument.update({ - where: { id: row.id }, - data: { deletedAt: null, status: restoredStatus(row.facts) }, - }); - } catch (err) { - if (isP2002(err)) { - // Raced a concurrent identical upload into the partial unique index. - return apiError("An identical document is already stored.", 409, { - errorCode: "documents.inbound.restoreDuplicate", - reason: "duplicateExists", - }); - } - throw err; } const restored = await prisma.inboundDocument.findFirstOrThrow({ - where: { id: row.id }, + where: { id, userId: user.id }, omit: { contentEncrypted: true }, include: { facts: { orderBy: { createdAt: "asc" } } }, }); @@ -150,15 +85,19 @@ export const POST = apiHandler( loadDocumentEncounterLinks(user.id, [restored.id]), ]); - await auditLog("documents.inbound.restore", { - userId: user.id, - ipAddress: getClientIp(request), - details: { documentId: restored.id }, - }); + // Restoring an already-live document is a no-op success (the undo toast + // may fire after a concurrent restore); only a real restore is audited. + if (outcome.kind === "restored") { + await auditLog("documents.inbound.restore", { + userId: user.id, + ipAddress: getClientIp(request), + details: { documentId: restored.id }, + }); + } annotate({ action: { name: "documents.vault.restore" }, - meta: { documentId: restored.id }, + meta: { documentId: restored.id, alreadyLive: outcome.kind === "live" }, }); return apiSuccess( diff --git a/src/app/api/documents/inbound/bulk/route.ts b/src/app/api/documents/inbound/bulk/route.ts index dbd3591c4..8ad9cdf74 100644 --- a/src/app/api/documents/inbound/bulk/route.ts +++ b/src/app/api/documents/inbound/bulk/route.ts @@ -17,6 +17,7 @@ import { apiError, apiSuccess, getClientIp } from "@/lib/api-response"; import { parseJsonBody } from "@/lib/api/body"; import { auditLog } from "@/lib/auth/audit"; import { prisma } from "@/lib/db"; +import { restoreDocumentTombstone } from "@/lib/documents/tombstones"; import { linkTargets, unlinkTargets } from "@/lib/links"; import { annotate } from "@/lib/logging/context"; import { requireModuleEnabled } from "@/lib/modules/gate"; @@ -95,7 +96,7 @@ export const POST = apiHandler(async (request: NextRequest) => { // then never touches a row the caller does not own. const owned = await prisma.inboundDocument.findMany({ where: { id: { in: uniqueIds }, userId: user.id }, - select: { id: true, deletedAt: true, contentSha256: true }, + select: { id: true, deletedAt: true }, }); const byId = new Map(owned.map((d) => [d.id, d])); @@ -182,39 +183,18 @@ export const POST = apiHandler(async (request: NextRequest) => { break; } case "restore": { - if (row.deletedAt === null) { - // Already live — restoring twice is a no-op success. - break; + // The shared restore: the duplicate check, the status the staged + // facts imply, and the import-ledger cleanup. Already live is a + // no-op success. + const outcome = await restoreDocumentTombstone(user.id, id); + if (outcome.kind === "gone") { + results.push({ id, ok: false, error: "notFound" }); + continue; } - if (row.contentSha256) { - const conflict = await prisma.inboundDocument.findFirst({ - where: { - userId: user.id, - contentSha256: row.contentSha256, - deletedAt: null, - id: { not: id }, - }, - select: { id: true }, - }); - if (conflict) { - results.push({ id, ok: false, error: "conflict" }); - continue; - } + if (outcome.kind === "duplicate") { + results.push({ id, ok: false, error: "conflict" }); + continue; } - const facts = await prisma.extractedFact.findMany({ - where: { documentId: id, userId: user.id }, - select: { status: true }, - }); - const status = - facts.length === 0 - ? "STORED" - : facts.some((f) => f.status === "PENDING") - ? "EXTRACTED" - : "CONFIRMED"; - await prisma.inboundDocument.update({ - where: { id }, - data: { deletedAt: null, status }, - }); break; } } diff --git a/src/app/api/documents/inbound/deleted/route.ts b/src/app/api/documents/inbound/deleted/route.ts new file mode 100644 index 000000000..fb0a9fde5 --- /dev/null +++ b/src/app/api/documents/inbound/deleted/route.ts @@ -0,0 +1,92 @@ +/** + * Document vault: Recently deleted. + * + * GET lists the caller's deleted documents that the purge has not removed yet, + * newest deletion first, each with the moment it becomes due for the purge. + * Restoring one is `POST /api/documents/inbound/{id}/restore` (or the bulk + * route's `restore`); removing one for good ahead of time is + * `POST /api/documents/inbound/{id}/purge`. + * + * Owner-only, like the restore and delete beside it: a deleted document is the + * owner's decision in progress, and nothing a delegate may do (restore, delete + * now) is a verb a grant admits. A bare `requireAuth()` therefore refuses the + * request while it acts inside somebody else's record, and refuses a narrow + * token. + * + * Metadata only. The encrypted blob is never selected; a deleted document's + * original, thumbnail and AI surfaces stay closed until it is restored. + */ +import { apiHandler, requireAuth } from "@/lib/api-handler"; +import { apiListSuccess } from "@/lib/api/list-cap"; +import { prisma } from "@/lib/db"; +import { DOCUMENT_PURGE_GRACE_DAYS } from "@/lib/documents/inbound-constants"; +import { documentPurgeAt } from "@/lib/documents/tombstones"; +import { listPage } from "@/lib/http/list-query"; +import { annotate } from "@/lib/logging/context"; +import { requireModuleEnabled } from "@/lib/modules/gate"; +import { dateOnlyKey } from "@/lib/tz/date-only"; +import { + toDocumentSourceSystem, + type DeletedDocumentListDto, + type InboundDocumentKindValue, +} from "@/lib/validations/inbound-documents"; + +export const dynamic = "force-dynamic"; + +/** + * Rows per answer. Thirty days of deletions rarely come near it; a person who + * cleared out a large import gets the newest ones and the list-cap notice. + */ +const DELETED_DOCUMENTS_LIMIT = 200; + +export const GET = apiHandler(async () => { + const { user } = await requireAuth(); + const gate = await requireModuleEnabled(user.id, "inboundDocuments"); + if (!gate.enabled) return gate.response; + + const { rows, cap } = await listPage(DELETED_DOCUMENTS_LIMIT, (take) => + prisma.inboundDocument.findMany({ + where: { userId: user.id, deletedAt: { not: null } }, + select: { + id: true, + kind: true, + title: true, + filename: true, + mimeType: true, + byteSize: true, + documentDate: true, + sourceSystem: true, + deletedAt: true, + }, + orderBy: [{ deletedAt: "desc" }, { id: "asc" }], + take, + }), + ); + + const payload: DeletedDocumentListDto = { + graceDays: DOCUMENT_PURGE_GRACE_DAYS, + documents: rows.map((row) => { + // Selected under `deletedAt: { not: null }`. + const deletedAt = row.deletedAt as Date; + return { + id: row.id, + kind: row.kind as InboundDocumentKindValue, + title: row.title, + filename: row.filename, + mimeType: row.mimeType, + byteSize: row.byteSize, + documentDate: row.documentDate ? dateOnlyKey(row.documentDate) : null, + sourceSystem: toDocumentSourceSystem(row.sourceSystem), + deletedAt: deletedAt.toISOString(), + purgeAt: documentPurgeAt(deletedAt).toISOString(), + }; + }), + }; + + annotate({ + action: { name: "documents.trash.list" }, + meta: { count: rows.length, truncated: cap.truncated }, + }); + + return apiListSuccess(payload, cap); +}); diff --git a/src/app/api/documents/inbound/usage/route.ts b/src/app/api/documents/inbound/usage/route.ts index 74a96dc1c..83a9be9c1 100644 --- a/src/app/api/documents/inbound/usage/route.ts +++ b/src/app/api/documents/inbound/usage/route.ts @@ -57,6 +57,7 @@ export const GET = apiHandler(async () => { capability, indexedCount, totalCount, + deletedCount, ] = await Promise.all([ resolveDocumentLimits(user.id), prisma.$queryRaw>` @@ -93,6 +94,15 @@ export const GET = apiHandler(async () => { prisma.inboundDocument.count({ where: { userId: user.id, deletedAt: null }, }), + // How many sit in Recently deleted — the vault's way in to it shows only + // when there is something there. Recently deleted is the owner's own + // (restore and delete-now are owner-only writes), so a delegate gets 0 + // and the vault offers them no way in. + grantId === null + ? prisma.inboundDocument.count({ + where: { userId: user.id, deletedAt: { not: null } }, + }) + : Promise.resolve(0), ]); const usedBytes = Number(rows[0]?.used ?? 0); @@ -121,6 +131,7 @@ export const GET = apiHandler(async () => { indexedCount, totalCount, }, + deletedCount, }; return apiSuccess(payload); }); diff --git a/src/app/documents/deleted/page.tsx b/src/app/documents/deleted/page.tsx new file mode 100644 index 000000000..5f35e7761 --- /dev/null +++ b/src/app/documents/deleted/page.tsx @@ -0,0 +1,9 @@ +import { RecentlyDeletedView } from "@/components/documents/trash/recently-deleted-view"; + +/** + * Recently deleted — server shell only. The client surface owns the module + * gate, the owner-only redirect and the list; the AuthShell owns the frame. + */ +export default function RecentlyDeletedPage() { + return ; +} diff --git a/src/components/documents/detail/use-document-detail.ts b/src/components/documents/detail/use-document-detail.ts index 5271f7b08..ff269a6d2 100644 --- a/src/components/documents/detail/use-document-detail.ts +++ b/src/components/documents/detail/use-document-detail.ts @@ -15,6 +15,7 @@ import { apiPatch, apiPost, } from "@/lib/api/api-fetch"; +import { DOCUMENT_PURGE_GRACE_DAYS } from "@/lib/documents/inbound-constants"; import { showErrorToast } from "@/lib/errors/error-toast"; import { useTranslations } from "@/lib/i18n/context"; import { invalidateKeys, queryKeys } from "@/lib/query-keys"; @@ -304,6 +305,9 @@ function useDocumentWrites({ onOpenChange(false); void invalidateKeys(queryClient, [queryKeys.documents()]); toast.success(t("documents.toast.deleted"), { + description: t("documents.toast.deletedHint", { + days: DOCUMENT_PURGE_GRACE_DAYS, + }), action: { label: t("common.undo"), onClick: () => restore.mutate(id), diff --git a/src/components/documents/document-detail-sheet.tsx b/src/components/documents/document-detail-sheet.tsx index ce9e69966..25e3cd02a 100644 --- a/src/components/documents/document-detail-sheet.tsx +++ b/src/components/documents/document-detail-sheet.tsx @@ -26,6 +26,7 @@ import { QueryErrorRow } from "@/components/ui/query-error-row"; import { ResponsiveSheet } from "@/components/ui/responsive-sheet"; import { Skeleton } from "@/components/ui/skeleton"; import { VaccinationDocumentSuggestion } from "@/components/vaccinations/vaccination-document-suggestion"; +import { DOCUMENT_PURGE_GRACE_DAYS } from "@/lib/documents/inbound-constants"; import { formatBytes } from "@/lib/format/bytes"; import { useFormatters, useTranslations } from "@/lib/i18n/context"; import type { RelatedLink } from "@/lib/links/related"; @@ -45,6 +46,9 @@ import { DocumentRecordLinks } from "./document-record-links"; import { DocumentShareSheet } from "./document-share-sheet"; import { DocumentSummaryBlock } from "./document-summary-block"; +/** The grace window the delete confirmation names. */ +const GRACE = { days: DOCUMENT_PURGE_GRACE_DAYS }; + export function DocumentDetailSheet({ documentId, open, @@ -359,7 +363,7 @@ export function DocumentDetailSheet({ open={confirmDeleteOpen} onOpenChange={setConfirmDeleteOpen} title={t("documents.deleteConfirm.titleOne")} - body={t("documents.deleteConfirm.body")} + body={t("documents.deleteConfirm.body", GRACE)} confirmLabel={t("documents.deleteConfirm.action")} onConfirm={() => remove.mutate(doc.id)} pending={remove.isPending} diff --git a/src/components/documents/documents-view.tsx b/src/components/documents/documents-view.tsx index da5c67a9d..d3084ee61 100644 --- a/src/components/documents/documents-view.tsx +++ b/src/components/documents/documents-view.tsx @@ -29,6 +29,7 @@ import { QueryErrorCard } from "@/components/ui/query-error-card"; import { Skeleton } from "@/components/ui/skeleton"; import { useAuth } from "@/hooks/use-auth"; import { useRecordCapabilities } from "@/hooks/use-record-capabilities"; +import { DOCUMENT_PURGE_GRACE_DAYS } from "@/lib/documents/inbound-constants"; import { useTranslations } from "@/lib/i18n/context"; import { DEFAULT_TIMEZONE } from "@/lib/tz/format"; @@ -46,6 +47,9 @@ import { useVaultSelection } from "./vault/use-vault-selection"; import { VaultDropOverlay } from "./vault/vault-drop-overlay"; import { VaultPageActions } from "./vault/vault-page-actions"; +/** The grace window the delete confirmation names. */ +const GRACE = { days: DOCUMENT_PURGE_GRACE_DAYS }; + /** * The vault page. Its state lives in `./vault/`: the URL filters, the * library (list, usage, uploads, filter choices), the selection and its bulk @@ -334,7 +338,7 @@ export function DocumentsView() { }) : t("documents.deleteConfirm.titleOne") } - body={t("documents.deleteConfirm.body")} + body={t("documents.deleteConfirm.body", GRACE)} confirmLabel={t("documents.deleteConfirm.action")} onConfirm={confirmDelete} slot="documents-delete" diff --git a/src/components/documents/trash/__tests__/recently-deleted-view.test.tsx b/src/components/documents/trash/__tests__/recently-deleted-view.test.tsx new file mode 100644 index 000000000..33d7d92d3 --- /dev/null +++ b/src/components/documents/trash/__tests__/recently-deleted-view.test.tsx @@ -0,0 +1,186 @@ +import { describe, expect, it, vi } from "vitest"; +import { renderToStaticMarkup } from "react-dom/server"; +import { createRef } from "react"; +import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; + +/** + * Recently deleted and the vault's way into it, pinned via single-pass static + * renders (the project's component-test convention): + * + * - the page paints a skeleton while the list is out, never the empty copy; + * - each row says what the document was, where it came from, when it was + * deleted and how long it has left, and offers Restore and Delete now; + * - Restore all appears only with more than one document; + * - an empty list says so and leads back to the vault; + * - the vault header links to Recently deleted with the count, and only + * while something is in it. + */ + +vi.mock("next/navigation", () => ({ + useRouter: () => ({ push: vi.fn(), replace: vi.fn() }), + usePathname: () => "/documents/deleted", + useSearchParams: () => new URLSearchParams(), +})); + +vi.mock("@/hooks/use-auth", () => ({ + useAuth: () => ({ + user: { id: "u1", modules: { inboundDocuments: true } }, + isLoading: false, + isAuthenticated: true, + }), +})); + +vi.mock("@/lib/api/api-fetch", async () => { + const actual = await vi.importActual( + "@/lib/api/api-fetch", + ); + return { + ...actual, + // Never resolves within the synchronous render: an unseeded query stays + // pending. + apiGet: () => new Promise(() => {}), + apiGetList: () => new Promise(() => {}), + apiPost: vi.fn(), + }; +}); + +import { I18nProvider } from "@/lib/i18n/context"; +import { queryKeys } from "@/lib/query-keys"; +import type { DeletedDocumentDto } from "@/lib/validations/inbound-documents"; + +import { VaultPageActions } from "../../vault/vault-page-actions"; +import { RecentlyDeletedView } from "../recently-deleted-view"; + +const DAY = 86_400_000; + +function doc(over: Partial = {}): DeletedDocumentDto { + const deletedAt = new Date(Date.now() - 3 * DAY); + return { + id: "doc-1", + kind: "LAB_RESULT", + title: "Blood panel", + filename: "blood.pdf", + mimeType: "application/pdf", + byteSize: 1234, + documentDate: "2024-03-04", + sourceSystem: "PAPERLESS", + deletedAt: deletedAt.toISOString(), + purgeAt: new Date(deletedAt.getTime() + 30 * DAY).toISOString(), + ...over, + }; +} + +function render(node: React.ReactNode, seed?: (qc: QueryClient) => void) { + const queryClient = new QueryClient({ + defaultOptions: { queries: { retry: false } }, + }); + seed?.(queryClient); + return renderToStaticMarkup( + + {node} + , + ); +} + +function seedList(documents: DeletedDocumentDto[], truncated = false) { + return (qc: QueryClient) => + qc.setQueryData(queryKeys.inboundDocumentsDeleted(), { + data: { documents, graceDays: 30 }, + cap: { truncated, limit: 200 }, + }); +} + +describe("", () => { + it("first load: header and a skeleton, never the empty copy", () => { + const html = render(); + expect(html).toContain("Recently deleted"); + expect(html).toContain( + "Deleted documents stay here for 30 days, then they are removed for good.", + ); + expect(html).toContain('data-slot="documents-trash-loading"'); + expect(html).toContain('href="/documents"'); + expect(html).not.toContain("Nothing in Recently deleted"); + }); + + it("a row: what it was, where it came from, how long it has left, both ways out", () => { + const html = render(, seedList([doc()])); + expect(html).toContain('data-slot="documents-trash-card"'); + expect(html).toContain("Deleted documents (1)"); + expect(html).toContain("Blood panel"); + expect(html).toContain("Lab result"); + expect(html).toContain("Imported from Paperless-ngx"); + expect(html).toContain("Removed for good in 27 days."); + expect(html).toContain('data-slot="documents-trash-restore"'); + expect(html).toContain('data-slot="documents-trash-purge"'); + expect(html).toContain("Delete now"); + // One document: nothing to restore "all" of. + expect(html).not.toContain('data-slot="documents-trash-restore-all"'); + }); + + it("falls back to the file name, and says the last day plainly", () => { + const html = render( + , + seedList([ + doc({ + title: null, + sourceSystem: null, + purgeAt: new Date(Date.now() + 3_600_000).toISOString(), + }), + ]), + ); + expect(html).toContain("blood.pdf"); + expect(html).not.toContain("Imported from"); + expect(html).toContain("Removed for good within a day."); + }); + + it("offers Restore all with more than one document, and the cap notice when cut", () => { + const html = render( + , + seedList([doc(), doc({ id: "doc-2", title: "X-ray" })], true), + ); + expect(html).toContain('data-slot="documents-trash-restore-all"'); + expect(html).toContain("Restore all"); + expect(html).toContain("Deleted documents (2)"); + expect(html).toContain('data-slot="list-cap-notice"'); + }); + + it("empty: says so and leads back to the vault", () => { + const html = render(, seedList([])); + expect(html).toContain("Nothing in Recently deleted"); + expect(html).not.toContain('data-slot="documents-trash-card"'); + }); +}); + +describe(" — the way into Recently deleted", () => { + const actions = (inSharedRecord = false) => ( + ()} + /> + ); + const seedUsage = (deletedCount: number) => (qc: QueryClient) => + qc.setQueryData(queryKeys.inboundDocumentUsage(), { deletedCount }); + + it("links to Recently deleted with the count while something is in it", () => { + const html = render(actions(), seedUsage(3)); + expect(html).toContain('data-slot="documents-recently-deleted"'); + expect(html).toContain('href="/documents/deleted"'); + expect(html).toContain('aria-label="Recently deleted (3)"'); + expect(html).toContain("(3)"); + }); + + it("shows nothing while Recently deleted is empty or the usage is out", () => { + expect(render(actions(), seedUsage(0))).not.toContain( + "documents-recently-deleted", + ); + expect(render(actions())).not.toContain("documents-recently-deleted"); + }); + + it("shows nothing inside somebody else's record", () => { + expect(render(actions(true), seedUsage(3))).not.toContain( + "documents-recently-deleted", + ); + }); +}); diff --git a/src/components/documents/trash/__tests__/trash-utils.test.ts b/src/components/documents/trash/__tests__/trash-utils.test.ts new file mode 100644 index 000000000..e73dec3cd --- /dev/null +++ b/src/components/documents/trash/__tests__/trash-utils.test.ts @@ -0,0 +1,41 @@ +import { describe, expect, it } from "vitest"; + +import { daysUntilPurge, purgeCountdownKey } from "../trash-utils"; + +const NOW = new Date("2026-10-24T12:00:00Z"); + +describe("daysUntilPurge", () => { + it("rounds a part day up: the whole of it is still restorable", () => { + expect(daysUntilPurge("2026-11-20T15:00:00Z", NOW)).toBe(28); + expect(daysUntilPurge("2026-10-25T11:00:00Z", NOW)).toBe(1); + expect(daysUntilPurge("2026-10-24T12:00:01Z", NOW)).toBe(1); + }); + + it("is zero once the moment has passed, and for an unreadable date", () => { + expect(daysUntilPurge("2026-10-24T12:00:00Z", NOW)).toBe(0); + expect(daysUntilPurge("2026-10-01T00:00:00Z", NOW)).toBe(0); + expect(daysUntilPurge("not a date", NOW)).toBe(0); + }); +}); + +describe("purgeCountdownKey", () => { + it("names the days while there are two or more", () => { + expect(purgeCountdownKey(30)).toEqual({ + key: "documents.trash.removesIn", + params: { days: 30 }, + }); + expect(purgeCountdownKey(2)).toEqual({ + key: "documents.trash.removesIn", + params: { days: 2 }, + }); + }); + + it("says within a day for the last one, and after the moment passed", () => { + expect(purgeCountdownKey(1)).toEqual({ + key: "documents.trash.removesSoon", + }); + expect(purgeCountdownKey(0)).toEqual({ + key: "documents.trash.removesSoon", + }); + }); +}); diff --git a/src/components/documents/trash/recently-deleted-view.tsx b/src/components/documents/trash/recently-deleted-view.tsx new file mode 100644 index 000000000..865020ed3 --- /dev/null +++ b/src/components/documents/trash/recently-deleted-view.tsx @@ -0,0 +1,270 @@ +"use client"; + +/** + * Recently deleted — the vault's deleted documents, until the nightly purge + * removes them for good. + * + * Deleting in the vault only sets a tombstone; for 30 days the document can + * come back. Before this page the only way back was the undo toast, which is + * gone a few seconds after the delete. Each row says what the document was + * (title, type, filing date, where it came from), when it was deleted and how + * long it has left, and offers Restore and "Delete now". Restoring all of them + * is one action in the card header. + * + * Owner-only, like every route behind it: inside somebody else's record the + * page sends the person back to the vault, which offers no way in either. + */ +import { useEffect, useState } from "react"; +import Link from "next/link"; +import { useRouter } from "next/navigation"; +import { Loader2, RotateCcw, Trash2 } from "lucide-react"; + +import { TileHeader } from "@/components/insights/tile-header"; +import { Button } from "@/components/ui/button"; +import { Card, CardHeader } from "@/components/ui/card"; +import { ConfirmDialog } from "@/components/ui/confirm-button"; +import { EmptyState } from "@/components/ui/empty-state"; +import { InlineAction } from "@/components/ui/inline-action"; +import { ListCapNotice } from "@/components/ui/list-cap-notice"; +import { PageHeader } from "@/components/ui/page-header"; +import { QueryErrorCard } from "@/components/ui/query-error-card"; +import { Skeleton } from "@/components/ui/skeleton"; +import { useAuth } from "@/hooks/use-auth"; +import { useRecordCapabilities } from "@/hooks/use-record-capabilities"; +import { DOCUMENT_PURGE_GRACE_DAYS } from "@/lib/documents/inbound-constants"; +import { useFormatters, useTranslations } from "@/lib/i18n/context"; +import type { + DeletedDocumentDto, + DocumentSourceSystemValue, +} from "@/lib/validations/inbound-documents"; + +import { DOCUMENT_KIND_ICONS } from "../document-kind-meta"; + +import { daysUntilPurge, purgeCountdownKey } from "./trash-utils"; +import { useRecentlyDeleted } from "./use-recently-deleted"; + +export function RecentlyDeletedView() { + const { t } = useTranslations(); + const { user, isLoading: authLoading, isAuthenticated } = useAuth(); + const { inSharedRecord } = useRecordCapabilities(); + const router = useRouter(); + const moduleEnabled = user?.modules?.inboundDocuments === true; + const ready = !authLoading && isAuthenticated && moduleEnabled; + + // UX redirects only; every route behind the page enforces the same. + useEffect(() => { + if (authLoading) return; + if (!isAuthenticated) router.push("/auth/login"); + else if (!moduleEnabled) router.push("/"); + else if (inSharedRecord) router.push("/documents"); + }, [authLoading, isAuthenticated, moduleEnabled, inSharedRecord, router]); + + const { list, restore, restoreAll, purge } = useRecentlyDeleted({ + enabled: ready && !inSharedRecord, + }); + const [pendingPurge, setPendingPurge] = useState( + null, + ); + + if (!ready || inSharedRecord) { + return ( +
+ + {t("nav.loadingScreen")} +
+ ); + } + + const documents = list.data?.data.documents ?? []; + const graceDays = list.data?.data.graceDays ?? DOCUMENT_PURGE_GRACE_DAYS; + + return ( +
+ + + {list.isPending ? ( +
+ {Array.from({ length: 3 }, (_, i) => ( + + ))} +
+ ) : list.isError ? ( + void list.refetch()} + /> + ) : documents.length === 0 ? ( + } + title={t("documents.trash.emptyTitle")} + description={t("documents.trash.emptyDescription", { + days: graceDays, + })} + action={ + + } + /> + ) : ( + + + 1 ? ( + + restoreAll.mutate(documents.map((d) => d.id)) + } + > + + {t("documents.trash.restoreAll")} + + ) : null + } + /> + +
    + {documents.map((doc) => ( + restore.mutate(doc.id)} + onPurge={() => setPendingPurge(doc)} + /> + ))} +
+ + + )} + + { + if (!open && !purge.isPending) setPendingPurge(null); + }} + title={t("documents.trash.purgeConfirmTitle")} + body={t("documents.trash.purgeConfirmBody")} + confirmLabel={t("documents.trash.purge")} + pending={purge.isPending} + holdOpen + destructive + slot="documents-trash-purge" + onConfirm={() => { + if (!pendingPurge) return; + purge.mutate(pendingPurge.id, { + onSettled: () => setPendingPurge(null), + }); + }} + /> +
+ ); +} + +/** Product names stay untranslated; only "another system" is prose. */ +function sourceName( + system: DocumentSourceSystemValue, + t: (key: string) => string, +): string { + if (system === "PAPERLESS") return "Paperless-ngx"; + if (system === "PAPRA") return "Papra"; + return t("documents.detail.sourceOther"); +} + +/** One deleted document: what it was, how long it has left, the two ways out. */ +export function DeletedRow({ + doc, + busy, + onRestore, + onPurge, +}: { + doc: DeletedDocumentDto; + busy: boolean; + onRestore: () => void; + onPurge: () => void; +}) { + const { t } = useTranslations(); + const fmt = useFormatters(); + const Icon = DOCUMENT_KIND_ICONS[doc.kind]; + const title = doc.title ?? doc.filename ?? t("documents.card.untitled"); + const about = [ + t(`documents.kind.${doc.kind}`), + doc.documentDate ? fmt.dateShortSmartCalendar(doc.documentDate) : null, + doc.sourceSystem + ? t("documents.detail.importedFrom", { + system: sourceName(doc.sourceSystem, t), + }) + : null, + ] + .filter(Boolean) + .join(", "); + const countdown = purgeCountdownKey(daysUntilPurge(doc.purgeAt)); + + return ( +
  • +
    + +
    +

    {title}

    +

    {about}

    +

    + {t("documents.trash.deletedOn", { + date: fmt.dateShortSmart(doc.deletedAt), + })}{" "} + {t(countdown.key, countdown.params)} +

    +
    +
    +
    + + +
    +
    • + ); +} diff --git a/src/components/documents/trash/trash-utils.ts b/src/components/documents/trash/trash-utils.ts new file mode 100644 index 000000000..9ff006ce7 --- /dev/null +++ b/src/components/documents/trash/trash-utils.ts @@ -0,0 +1,31 @@ +/** + * How long a document in Recently deleted has left, in whole days, read from + * the server's `purgeAt`. + * + * Rounded up: a document due in 26 days and 3 hours still has 27 days in which + * it can be restored. Zero once the moment has passed; the nightly purge then + * takes it at its next run. + */ +const DAY_MS = 86_400_000; + +export function daysUntilPurge( + purgeAt: string, + now: Date = new Date(), +): number { + const left = new Date(purgeAt).getTime() - now.getTime(); + if (!Number.isFinite(left) || left <= 0) return 0; + return Math.ceil(left / DAY_MS); +} + +/** + * The translation key and parameters for a row's countdown line. The purge + * runs once a night, so the last day is said as "within a day" rather than a + * time the job does not keep. + */ +export function purgeCountdownKey(days: number): { + key: "documents.trash.removesIn" | "documents.trash.removesSoon"; + params?: { days: number }; +} { + if (days <= 1) return { key: "documents.trash.removesSoon" }; + return { key: "documents.trash.removesIn", params: { days } }; +} diff --git a/src/components/documents/trash/use-recently-deleted.ts b/src/components/documents/trash/use-recently-deleted.ts new file mode 100644 index 000000000..139fca599 --- /dev/null +++ b/src/components/documents/trash/use-recently-deleted.ts @@ -0,0 +1,115 @@ +"use client"; + +import { useCallback } from "react"; +import { useRouter } from "next/navigation"; +import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query"; +import { toast } from "sonner"; + +import { toastWrittenOutcome } from "@/components/outcome/outcome-toast"; +import { ApiError, apiGetList, apiPost } from "@/lib/api/api-fetch"; +import { DOCUMENT_BULK_MAX_IDS } from "@/lib/documents/inbound-constants"; +import { showErrorToast } from "@/lib/errors/error-toast"; +import { useTranslations } from "@/lib/i18n/context"; +import { invalidateKeys, queryKeys } from "@/lib/query-keys"; +import type { + DeletedDocumentListDto, + DocumentBulkResultDto, +} from "@/lib/validations/inbound-documents"; + +/** + * Recently deleted: the list, and the three ways out of it. Restore brings a + * document back (one at a time, or all through the bulk route); "Delete now" + * removes one for good ahead of the grace window. Every write invalidates the + * `["documents"]` prefix, so the vault, its usage figures and this list repaint + * together. + */ +export function useRecentlyDeleted({ enabled }: { enabled: boolean }) { + const { t } = useTranslations(); + const queryClient = useQueryClient(); + const router = useRouter(); + + const list = useQuery({ + queryKey: queryKeys.inboundDocumentsDeleted(), + queryFn: () => + apiGetList("/api/documents/inbound/deleted"), + enabled, + }); + + const refresh = useCallback( + () => void invalidateKeys(queryClient, [queryKeys.documents()]), + [queryClient], + ); + + const restore = useMutation({ + mutationFn: (id: string) => + apiPost(`/api/documents/inbound/${id}/restore`, {}), + onSuccess: (_data, id) => { + refresh(); + toast.success(t("documents.trash.restored"), { + action: { + label: t("documents.trash.openRestored"), + onClick: () => + router.push(`/documents?doc=${encodeURIComponent(id)}`), + }, + }); + }, + onError: (error) => { + refresh(); + const reason = + error instanceof ApiError && typeof error.meta?.reason === "string" + ? error.meta.reason + : ""; + if (reason === "purged") { + showErrorToast(t("documents.error.purged"), { cause: error }); + } else if (reason === "duplicateExists") { + showErrorToast(t("documents.trash.restoreDuplicate"), { + cause: error, + }); + } else { + showErrorToast(t("documents.toast.restoreFailed"), { cause: error }); + } + }, + }); + + const restoreAll = useMutation({ + mutationFn: async (ids: string[]) => { + const results: DocumentBulkResultDto[] = []; + for (let i = 0; i < ids.length; i += DOCUMENT_BULK_MAX_IDS) { + const page = await apiPost<{ results: DocumentBulkResultDto[] }>( + "/api/documents/inbound/bulk", + { ids: ids.slice(i, i + DOCUMENT_BULK_MAX_IDS), action: "restore" }, + ); + results.push(...page.results); + } + return results; + }, + onSuccess: (results) => { + const restored = results.filter((r) => r.ok).length; + const failed = results.length - restored; + toastWrittenOutcome( + failed === 0 ? "success" : restored > 0 ? "partial" : "failed", + failed === 0 + ? t("documents.bulk.restored", { count: restored }) + : t("documents.trash.restorePartial", { + failed, + total: results.length, + }), + ); + }, + onError: (error) => + showErrorToast(t("documents.toast.restoreFailed"), { cause: error }), + onSettled: refresh, + }); + + const purge = useMutation({ + mutationFn: (id: string) => + apiPost(`/api/documents/inbound/${id}/purge`, {}), + onSuccess: () => + toastWrittenOutcome("success", t("documents.trash.purged")), + onError: (error) => + showErrorToast(t("documents.trash.purgeFailed"), { cause: error }), + onSettled: refresh, + }); + + return { list, restore, restoreAll, purge }; +} diff --git a/src/components/documents/vault/use-document-usage.ts b/src/components/documents/vault/use-document-usage.ts new file mode 100644 index 000000000..899a73ca5 --- /dev/null +++ b/src/components/documents/vault/use-document-usage.ts @@ -0,0 +1,21 @@ +"use client"; + +import { useQuery } from "@tanstack/react-query"; + +import { apiGet } from "@/lib/api/api-fetch"; +import { queryKeys } from "@/lib/query-keys"; +import type { DocumentUsageDto } from "@/lib/validations/inbound-documents"; + +/** + * The vault's usage read (`GET /api/documents/inbound/usage`): limits, usage, + * filter choices, index coverage and the Recently deleted count. One + * definition for every reader, so the shared key never meets a second + * `queryFn` shape. + */ +export function useDocumentUsage(enabled: boolean) { + return useQuery({ + queryKey: queryKeys.inboundDocumentUsage(), + enabled, + queryFn: () => apiGet("/api/documents/inbound/usage"), + }); +} diff --git a/src/components/documents/vault/use-vault-bulk.ts b/src/components/documents/vault/use-vault-bulk.ts index 875551303..4b5cff351 100644 --- a/src/components/documents/vault/use-vault-bulk.ts +++ b/src/components/documents/vault/use-vault-bulk.ts @@ -5,7 +5,10 @@ import { useMutation, useQueryClient } from "@tanstack/react-query"; import { toast } from "sonner"; import { apiPost } from "@/lib/api/api-fetch"; -import { DOCUMENT_BULK_MAX_IDS } from "@/lib/documents/inbound-constants"; +import { + DOCUMENT_BULK_MAX_IDS, + DOCUMENT_PURGE_GRACE_DAYS, +} from "@/lib/documents/inbound-constants"; import { showErrorToast } from "@/lib/errors/error-toast"; import { useTranslations } from "@/lib/i18n/context"; import { invalidateKeys, queryKeys } from "@/lib/query-keys"; @@ -155,6 +158,9 @@ export function useVaultBulk({ toast.success( t("documents.bulk.deleted", { count: okIds.length }), { + description: t("documents.toast.deletedHint", { + days: DOCUMENT_PURGE_GRACE_DAYS, + }), action: { label: t("common.undo"), onClick: () => restoreBulk(okIds), diff --git a/src/components/documents/vault/use-vault-library.ts b/src/components/documents/vault/use-vault-library.ts index c4afaa199..2e92034f4 100644 --- a/src/components/documents/vault/use-vault-library.ts +++ b/src/components/documents/vault/use-vault-library.ts @@ -1,7 +1,7 @@ "use client"; import { useCallback, useEffect, useMemo, useRef, useState } from "react"; -import { useInfiniteQuery, useQuery } from "@tanstack/react-query"; +import { useInfiniteQuery } from "@tanstack/react-query"; import { toast } from "sonner"; import { useIllnessEpisodes } from "@/components/illness/use-illness"; @@ -11,16 +11,14 @@ import { showErrorToast } from "@/lib/errors/error-toast"; import { useTranslations } from "@/lib/i18n/context"; import { useDocumentsLayout } from "@/lib/queries/use-documents-layout"; import { queryKeys } from "@/lib/query-keys"; -import type { - DocumentUsageDto, - InboundDocumentDto, -} from "@/lib/validations/inbound-documents"; +import type { InboundDocumentDto } from "@/lib/validations/inbound-documents"; import { useReindexAll } from "../use-content-index"; import { useDocumentUpload } from "../use-document-upload"; import { usePageFileDrop } from "../use-page-file-drop"; import { buildVaultListApiSearch, hasProcessingDocument } from "../vault-utils"; +import { useDocumentUsage } from "./use-document-usage"; import type { VaultFilters } from "./use-vault-filters"; import { useVaultFilterChoices } from "./use-vault-filter-choices"; @@ -51,11 +49,7 @@ export function useVaultLibrary({ timezone: string; }) { const { t } = useTranslations(); - const usage = useQuery({ - queryKey: queryKeys.inboundDocumentUsage(), - enabled: moduleEnabled, - queryFn: () => apiGet("/api/documents/inbound/usage"), - }); + const usage = useDocumentUsage(moduleEnabled); const list = useInfiniteQuery({ queryKey: queryKeys.inboundDocumentList(filters), diff --git a/src/components/documents/vault/vault-page-actions.tsx b/src/components/documents/vault/vault-page-actions.tsx index e2b511676..3b3f1bb3d 100644 --- a/src/components/documents/vault/vault-page-actions.tsx +++ b/src/components/documents/vault/vault-page-actions.tsx @@ -2,17 +2,19 @@ import type { RefObject } from "react"; import Link from "next/link"; -import { Upload, Wrench } from "lucide-react"; +import { Trash2, Upload, Wrench } from "lucide-react"; import { Button } from "@/components/ui/button"; import { useTranslations } from "@/lib/i18n/context"; import { ImportFromSourceButton } from "../sources/import-from-source-button"; +import { useDocumentUsage } from "./use-document-usage"; /** * The page header's actions (rendered only when one of them applies): the display-settings wrench (the caller's own - * settings, so never inside somebody else's record), the source import and - * the upload button, which an empty vault's empty state carries instead. + * settings, so never inside somebody else's record), the way into Recently + * deleted (only while something is in it), the source import and the upload + * button, which an empty vault's empty state carries instead. */ export function VaultPageActions({ canManageDocuments, @@ -26,6 +28,9 @@ export function VaultPageActions({ uploadInputRef: RefObject; }) { const { t } = useTranslations(); + // The vault's own usage read (shared key): how many sit in Recently + // deleted. The way in shows only above zero. + const deletedCount = useDocumentUsage(true).data?.deletedCount ?? 0; return (
    {/* The wrench left of the primary action, as on the illness @@ -49,6 +54,31 @@ export function VaultPageActions({ ) : null} + {/* Quiet, like the wrench, and only while there is something to + bring back: deleted documents wait there for 30 days. On a phone + the icon and the count; the words from `sm` up. */} + {canManageDocuments && !inSharedRecord && deletedCount > 0 ? ( + + ) : null} {canManageDocuments ? ( <> {/* Renders only with a connected Paperless-ngx or Papra. */} diff --git a/src/lib/documents/inbound-constants.ts b/src/lib/documents/inbound-constants.ts index be54767d5..37a2eca47 100644 --- a/src/lib/documents/inbound-constants.ts +++ b/src/lib/documents/inbound-constants.ts @@ -50,6 +50,9 @@ export const DOCUMENT_LIST_DEFAULT_LIMIT = 50; /** Max document ids one bulk request may touch. */ export const DOCUMENT_BULK_MAX_IDS = 100; +/** Days a deleted document stays in Recently deleted (see the purge queue). */ +export { DOCUMENT_PURGE_GRACE_DAYS } from "@/lib/jobs/document-purge-queue"; + /** Max documents attachable to one coach conversation. Bounds prompt size + the * numeric-grounding union; five labelled documents is already past the point a * chat answer stays useful. */ diff --git a/src/lib/documents/tombstones.ts b/src/lib/documents/tombstones.ts new file mode 100644 index 000000000..4a26f6f11 --- /dev/null +++ b/src/lib/documents/tombstones.ts @@ -0,0 +1,226 @@ +/** + * The two ways out of a deleted (tombstoned) vault document: back into the + * vault, or out of the database for good. + * + * Delete in the vault only stamps `deletedAt`. From there a document is either + * restored (the undo toast, Recently deleted, the bulk bar, the document + * picker) or purged (the daily job once the grace window has passed, or the + * person choosing "Delete now" in Recently deleted). Every one of those callers + * goes through this module, so a restore and a purge mean the same thing + * whoever asks: + * + * - a purge copies the document's import source keys (its own and every + * alias) into `DocumentImportKey` in the same transaction as the delete, + * so an importer re-sending the document is still answered "deleted" once + * the row is gone; + * - a restore removes any `DocumentImportKey` entry for those same keys in + * the same transaction as clearing the tombstone. The ledger says "the + * person deleted this"; once the document is back, that is no longer true, + * and a ledger entry left behind would outlive the document's next delete + * and restore with nothing to explain it. + */ +import type { InboundDocumentStatus, Prisma } from "@/generated/prisma/client"; +import { prisma } from "@/lib/db"; +import { DOCUMENT_PURGE_GRACE_DAYS } from "@/lib/documents/inbound-constants"; +import { isP2002 } from "@/lib/prisma-errors"; + +const DAY_MS = 86_400_000; + +/** The moment a document deleted at `deletedAt` becomes due for the purge. */ +export function documentPurgeAt(deletedAt: Date): Date { + return new Date(deletedAt.getTime() + DOCUMENT_PURGE_GRACE_DAYS * DAY_MS); +} + +interface ImportSourceKey { + userId: string; + sourceSystem: string; + sourceId: string; + sourceInstance: string | null; +} + +/** The columns a row needs for {@link importKeysOf}. */ +const SOURCE_KEY_SELECT = { + userId: true, + sourceSystem: true, + sourceId: true, + sourceInstance: true, + sourceAliases: { + select: { sourceSystem: true, sourceId: true, sourceInstance: true }, + }, +} as const; + +interface KeyedRow { + userId: string; + sourceSystem: string | null; + sourceId: string | null; + sourceInstance: string | null; + sourceAliases: Array<{ + sourceSystem: string; + sourceId: string; + sourceInstance: string | null; + }>; +} + +/** Every import source key a document holds: its own, then its aliases. */ +function importKeysOf(row: KeyedRow): ImportSourceKey[] { + return [ + ...(row.sourceSystem && row.sourceId + ? [ + { + userId: row.userId, + sourceSystem: row.sourceSystem, + sourceId: row.sourceId, + sourceInstance: row.sourceInstance, + }, + ] + : []), + ...row.sourceAliases.map((alias) => ({ + userId: row.userId, + sourceSystem: alias.sourceSystem, + sourceId: alias.sourceId, + sourceInstance: alias.sourceInstance, + })), + ]; +} + +/** + * Hard-delete the tombstones `where` selects, carrying their import source + * keys into the ledger first. Only rows with `deletedAt` set are ever touched, + * whatever `where` says. Runs inside the caller's transaction, so a purge that + * fails halfway forgets nothing. Returns the number of rows removed. + */ +export async function purgeDocumentTombstones( + tx: Prisma.TransactionClient, + where: Prisma.InboundDocumentWhereInput, +): Promise { + const tombstones: Prisma.InboundDocumentWhereInput = { + AND: [where, { deletedAt: { not: null } }], + }; + const keyed = await tx.inboundDocument.findMany({ + where: { + AND: [ + tombstones, + { + OR: [ + { sourceSystem: { not: null }, sourceId: { not: null } }, + { sourceAliases: { some: {} } }, + ], + }, + ], + }, + select: SOURCE_KEY_SELECT, + }); + const keys = keyed.flatMap(importKeysOf); + if (keys.length > 0) { + await tx.documentImportKey.createMany({ data: keys, skipDuplicates: true }); + } + const { count } = await tx.inboundDocument.deleteMany({ where: tombstones }); + return count; +} + +/** + * Purge one of the person's deleted documents now, ahead of the grace window. + * Returns false when there was no such tombstone (never existed, someone + * else's, still live, or already gone). + */ +export async function purgeDocumentTombstoneNow( + userId: string, + id: string, +): Promise { + const count = await prisma.$transaction((tx) => + purgeDocumentTombstones(tx, { id, userId }), + ); + return count > 0; +} + +export type TombstoneRestoreResult = + /** The tombstone was cleared. */ + | { kind: "restored" } + /** The document was live already; restoring twice changes nothing. */ + | { kind: "live" } + /** No such document for this person: purged, or never existed. */ + | { kind: "gone" } + /** A live document with the same bytes holds the slot (null on a race). */ + | { kind: "duplicate"; existingId: string | null }; + +/** + * Lifecycle status after a restore, derived from the staged facts: none → + * STORED, any PENDING → EXTRACTED, otherwise every fact was acted on → + * CONFIRMED. (Delete stamped DISCARDED; the status before it is not kept.) + */ +function restoredStatus( + facts: Array<{ status: string }>, +): InboundDocumentStatus { + if (facts.length === 0) return "STORED"; + return facts.some((f) => f.status === "PENDING") ? "EXTRACTED" : "CONFIRMED"; +} + +/** + * Bring one of the person's deleted documents back into the vault, and forget + * any ledger entry that says its import keys were deleted. Owner-scoped: a + * foreign id is `gone`, exactly like a purged one. + */ +export async function restoreDocumentTombstone( + userId: string, + id: string, +): Promise { + const row = await prisma.inboundDocument.findFirst({ + where: { id, userId }, + select: { id: true, deletedAt: true, contentSha256: true }, + }); + if (!row) return { kind: "gone" }; + if (row.deletedAt === null) return { kind: "live" }; + + // A live duplicate of the same bytes blocks the restore (the partial unique + // index would reject it; checked first for a clean answer). + if (row.contentSha256) { + const conflict = await prisma.inboundDocument.findFirst({ + where: { + userId, + contentSha256: row.contentSha256, + deletedAt: null, + id: { not: id }, + }, + select: { id: true }, + }); + if (conflict) return { kind: "duplicate", existingId: conflict.id }; + } + + const facts = await prisma.extractedFact.findMany({ + where: { documentId: id, userId }, + select: { status: true }, + }); + + let restored: boolean; + try { + restored = await prisma.$transaction(async (tx) => { + // Guarded on the tombstone, so a purge or a restore that won the race + // is not undone or repeated here. + const { count } = await tx.inboundDocument.updateMany({ + where: { id, userId, deletedAt: { not: null } }, + data: { deletedAt: null, status: restoredStatus(facts) }, + }); + if (count === 0) return false; + const keyed = await tx.inboundDocument.findUniqueOrThrow({ + where: { id }, + select: SOURCE_KEY_SELECT, + }); + const keys = importKeysOf(keyed); + if (keys.length > 0) { + await tx.documentImportKey.deleteMany({ where: { OR: keys } }); + } + return true; + }); + } catch (err) { + // Raced a concurrent identical upload into the partial unique index. + if (isP2002(err)) return { kind: "duplicate", existingId: null }; + throw err; + } + if (restored) return { kind: "restored" }; + + const now = await prisma.inboundDocument.findFirst({ + where: { id, userId }, + select: { deletedAt: true }, + }); + return now && now.deletedAt === null ? { kind: "live" } : { kind: "gone" }; +} diff --git a/src/lib/documents/vault-read-dtos.ts b/src/lib/documents/vault-read-dtos.ts new file mode 100644 index 000000000..2970d73e1 --- /dev/null +++ b/src/lib/documents/vault-read-dtos.ts @@ -0,0 +1,94 @@ +/** + * The payloads of the vault's side reads: the usage and limits the upload path + * pre-flights against (`GET /api/documents/inbound/usage`), and Recently + * deleted (`GET /api/documents/inbound/deleted`, `POST …/{id}/purge`). + * Re-exported from `@/lib/validations/inbound-documents`, where the other + * vault DTOs live. Types only. + */ +import type { + DocumentConditionLinkDto, + DocumentLinkedProcedureDto, + DocumentSourceSystemValue, + InboundDocumentKindValue, +} from "@/lib/validations/inbound-documents"; + +/** The usage endpoint payload the UI reads before offering an upload. */ +export interface DocumentUsageDto { + usedBytes: number; + quotaBytes: number; + maxFileBytes: number; + acceptedExtensions: string[]; + /** + * Episodes carrying at least one LIVE document link — the vault filter + * bar's condition chips. Sourced server-side so a chip exists even when + * every linked document sits pages deep in the timeline. + */ + linkedEpisodes: DocumentConditionLinkDto[]; + /** + * Procedures and surgeries (visits of kind PROCEDURE) carrying at least one + * LIVE document link — the filter bar's procedure choices, sourced + * server-side for the same reason as `linkedEpisodes`. Empty when the + * caller's grant does not cover the visits section, because the label is + * the visit's own text. + */ + linkedProcedures: DocumentLinkedProcedureDto[]; + /** + * Whether the AI "Suggest details" action can run for this caller — true when + * a provider (vision, or text + local OCR) is configured. The UI hides the + * action when false so it never offers what the endpoint would 422. + */ + assistAvailable: boolean; + /** + * Content-search index state: whether indexing is available to the caller + * (`enabled`, same provider precondition as assist), how many live documents + * are indexed, and the live-document total — so the UI can gauge coverage and + * offer the "index all documents" backfill. + */ + contentIndex: { + enabled: boolean; + indexedCount: number; + totalCount: number; + }; + /** + * How many of the person's documents sit in Recently deleted (deleted, not + * yet purged). The vault shows its way into Recently deleted only when this + * is above zero. Always 0 while acting inside somebody else's record, where + * Recently deleted is not offered. + */ + deletedCount: number; +} + +/** One document in Recently deleted (`GET /api/documents/inbound/deleted`). */ +export interface DeletedDocumentDto { + id: string; + kind: InboundDocumentKindValue; + title: string | null; + filename: string | null; + mimeType: string; + byteSize: number; + /** User-set filing date (YYYY-MM-DD), or null. */ + documentDate: string | null; + /** The system an imported document came from, or null. */ + sourceSystem: DocumentSourceSystemValue | null; + /** When it was deleted (ISO 8601). */ + deletedAt: string; + /** + * When the daily purge becomes due for it (ISO 8601): `deletedAt` plus the + * grace window. The purge runs once a day, so the row goes at the first run + * after this instant. + */ + purgeAt: string; +} + +/** The Recently deleted list, newest deletion first. */ +export interface DeletedDocumentListDto { + documents: DeletedDocumentDto[]; + /** Days a deleted document stays restorable. */ + graceDays: number; +} + +/** Answer of `POST /api/documents/inbound/{id}/purge`. */ +export interface DocumentPurgeDto { + id: string; + purged: true; +} diff --git a/src/lib/jobs/document-purge-queue.ts b/src/lib/jobs/document-purge-queue.ts new file mode 100644 index 000000000..f6d35dd7c --- /dev/null +++ b/src/lib/jobs/document-purge-queue.ts @@ -0,0 +1,17 @@ +/** + * The queue facts of `document-purge.ts`: its name, its schedule and the grace + * window it enforces. They live apart from the handler so the worker registry + * can read them without loading the purge (and the document modules it shares + * with "Delete now"). Constants only: client code reads the grace window + * through `@/lib/documents/inbound-constants`. + */ +export const DOCUMENT_PURGE_QUEUE = "document-tombstone-purge"; +/** Daily at 04:10 Europe/Berlin — after the med-notes backfill tick (04:05). */ +export const DOCUMENT_PURGE_CRON = "10 4 * * *"; + +/** + * Days a deleted document stays in Recently deleted (restorable) before the + * daily purge removes it for good. The purge, the list of deleted documents + * and the copy that names the window all read this one value. + */ +export const DOCUMENT_PURGE_GRACE_DAYS = 30; diff --git a/src/lib/jobs/document-purge.ts b/src/lib/jobs/document-purge.ts index d98e9a988..886ddef15 100644 --- a/src/lib/jobs/document-purge.ts +++ b/src/lib/jobs/document-purge.ts @@ -8,20 +8,21 @@ * tombstoned document keeps counting against the quota until purged, so * "deleted" bytes are never invisible backup weight). Staged facts cascade * via the FK. Restore-after-purge answers 409 at the API. An imported - * document's source key is carried to `DocumentImportKey` on the way out. + * document's source key is carried to `DocumentImportKey` on the way out + * (`src/lib/documents/tombstones.ts`, shared with "Delete now"). */ import { type Job } from "pg-boss"; +import { DOCUMENT_PURGE_GRACE_DAYS } from "@/lib/jobs/document-purge-queue"; +import { purgeDocumentTombstones } from "@/lib/documents/tombstones"; import { withBackgroundEvent } from "@/lib/logging/background"; import { jobDone, jobFailed, type JobOutcome } from "@/lib/jobs/job-outcome"; import { getWorkerPrisma } from "@/lib/jobs/reminder/shared"; import type { PrismaClient } from "@/generated/prisma/client"; -export const DOCUMENT_PURGE_QUEUE = "document-tombstone-purge"; -/** Daily at 04:10 Europe/Berlin — after the med-notes backfill tick (04:05). */ -export const DOCUMENT_PURGE_CRON = "10 4 * * *"; - -/** Days a tombstoned document stays restorable before the purge claims it. */ -export const DOCUMENT_PURGE_GRACE_DAYS = 30; +export { + DOCUMENT_PURGE_CRON, + DOCUMENT_PURGE_QUEUE, +} from "@/lib/jobs/document-purge-queue"; const DAY_MS = 86_400_000; @@ -33,66 +34,20 @@ export interface DocumentPurgePayload { * Hard-delete tombstoned `inbound_documents` rows whose `deletedAt` is older * than the grace horizon. Returns the number of rows purged. Rows inside the * grace window and live rows are never touched. + * + * v1.39.2 (#1038) — an imported document's source key outlives its row: the + * shared purge (`purgeDocumentTombstones`, the same one "Delete now" in + * Recently deleted runs) copies it into the ledger in the same transaction as + * the delete, so a re-run of the importer does not store the document again. */ export async function purgeExpiredDocumentTombstones( prisma: PrismaClient, now: Date = new Date(), ): Promise { const cutoff = new Date(now.getTime() - DOCUMENT_PURGE_GRACE_DAYS * DAY_MS); - const expired = { deletedAt: { not: null, lt: cutoff } }; - return prisma.$transaction(async (tx) => { - // v1.39.2 (#1038) — an imported document's source key outlives its row. - // While the tombstone exists the key sits on it and the upload route - // answers "deleted"; once the row goes, this ledger is what keeps a re-run - // of the importer from storing the document again. Same transaction as the - // delete, so a purge that fails halfway forgets nothing. - const keyed = await tx.inboundDocument.findMany({ - where: { - ...expired, - OR: [ - { sourceSystem: { not: null }, sourceId: { not: null } }, - { sourceAliases: { some: {} } }, - ], - }, - select: { - userId: true, - sourceSystem: true, - sourceId: true, - sourceInstance: true, - // Further keys the import answered with this document (same bytes). - // They cascade away with the row, so they are copied here first. - sourceAliases: { - select: { sourceSystem: true, sourceId: true, sourceInstance: true }, - }, - }, - }); - const keys = keyed.flatMap((row) => [ - ...(row.sourceSystem && row.sourceId - ? [ - { - userId: row.userId, - sourceSystem: row.sourceSystem, - sourceId: row.sourceId, - sourceInstance: row.sourceInstance, - }, - ] - : []), - ...row.sourceAliases.map((alias) => ({ - userId: row.userId, - sourceSystem: alias.sourceSystem, - sourceId: alias.sourceId, - sourceInstance: alias.sourceInstance, - })), - ]); - if (keys.length > 0) { - await tx.documentImportKey.createMany({ - data: keys, - skipDuplicates: true, - }); - } - const { count } = await tx.inboundDocument.deleteMany({ where: expired }); - return count; - }); + return prisma.$transaction((tx) => + purgeDocumentTombstones(tx, { deletedAt: { lt: cutoff } }), + ); } export async function handleDocumentPurge( diff --git a/src/lib/jobs/registry/maintenance.ts b/src/lib/jobs/registry/maintenance.ts index 146a43aa6..70429c156 100644 --- a/src/lib/jobs/registry/maintenance.ts +++ b/src/lib/jobs/registry/maintenance.ts @@ -70,7 +70,7 @@ import { import { DOCUMENT_PURGE_CRON, DOCUMENT_PURGE_QUEUE, -} from "@/lib/jobs/document-purge"; +} from "@/lib/jobs/document-purge-queue"; import { DOCUMENT_SUMMARY_CATCHUP_CONCURRENCY, DOCUMENT_SUMMARY_CATCHUP_QUEUE, diff --git a/src/lib/jobs/soft-delete-retention.ts b/src/lib/jobs/soft-delete-retention.ts index beab28258..e2a9e2bf0 100644 --- a/src/lib/jobs/soft-delete-retention.ts +++ b/src/lib/jobs/soft-delete-retention.ts @@ -36,7 +36,7 @@ import type { PrismaClient } from "@/generated/prisma/client"; import { TOMBSTONE_RETENTION_DAYS } from "@/lib/auth/native-client"; import { getCoachMessageRetentionDays } from "@/lib/jobs/coach-message-cleanup"; -import { DOCUMENT_PURGE_GRACE_DAYS } from "@/lib/jobs/document-purge"; +import { DOCUMENT_PURGE_GRACE_DAYS } from "@/lib/jobs/document-purge-queue"; import { purgeTombstonesByAccount, type TombstonePurgeOutcome, diff --git a/src/lib/openapi/error-codes.ts b/src/lib/openapi/error-codes.ts index 89f16b3ce..7433c5490 100644 --- a/src/lib/openapi/error-codes.ts +++ b/src/lib/openapi/error-codes.ts @@ -161,6 +161,7 @@ export const ERROR_CODE_CATALOGUE: Readonly> = "documents.inbound.invalidQuery", "documents.inbound.invalidUpdate", "documents.inbound.localOcrDisabled", + "documents.inbound.notDeleted", "documents.inbound.notFound", "documents.inbound.notIndexed", "documents.inbound.outputLimit", diff --git a/src/lib/openapi/routes/documents.ts b/src/lib/openapi/routes/documents.ts index fe1f1b054..9d870075f 100644 --- a/src/lib/openapi/routes/documents.ts +++ b/src/lib/openapi/routes/documents.ts @@ -44,6 +44,7 @@ import { errorEnvelope, idempotencyKeyParameter, idempotentWrite, + listEnvelope, recordRefusal, stdResponses, } from "./shared"; @@ -462,6 +463,47 @@ export const documentIndexResponse = z }) .meta({ id: "DocumentIndexResponse" }); +// ─── Recently deleted ────────────────────────────────────────────────────── + +const deletedDocument = z + .object({ + id: z.string(), + kind: kindEnum, + title: z.string().nullable(), + filename: z.string().nullable(), + mimeType: z.string(), + byteSize: z.number().int(), + documentDate: z + .string() + .nullable() + .describe("User-set filing date (YYYY-MM-DD), or null."), + sourceSystem: z + .enum(DOCUMENT_SOURCE_SYSTEMS) + .nullable() + .describe( + "The system an imported document came from, or null for one added in HealthLog.", + ), + deletedAt: isoDateTime(), + purgeAt: isoDateTime().describe( + "When the daily purge becomes due for this document: `deletedAt` plus `graceDays`. The purge runs once a day, so the document is removed at the first run after this instant.", + ), + }) + .meta({ id: "DeletedDocument" }); + +const deletedDocumentList = z + .object({ + documents: z.array(deletedDocument), + graceDays: z + .number() + .int() + .describe("Days a deleted document stays restorable."), + }) + .meta({ id: "DeletedDocumentList" }); + +const documentPurgeResult = z + .object({ id: z.string(), purged: z.literal(true) }) + .meta({ id: "DocumentPurgeResult" }); + export const inboundDocumentPaths: NonNullable = { "/api/documents/inbound": { get: { @@ -837,6 +879,12 @@ export const inboundDocumentPaths: NonNullable = { indexedCount: z.number(), totalCount: z.number(), }), + deletedCount: z + .number() + .int() + .describe( + "How many of the caller's documents are deleted and not yet purged (Recently deleted, `GET /api/documents/inbound/deleted`). Always 0 while acting inside somebody else's record, where Recently deleted is not offered.", + ), }) .meta({ id: "DocumentUsage" }), "DocumentUsageEnvelope", @@ -1027,7 +1075,7 @@ export const inboundDocumentPaths: NonNullable = { tags: ["Documents"], summary: "Restore a discarded document", description: - 'Clears the soft-delete tombstone within the 30-day undo grace. 409 when the tombstone is already purged (the undo window is over) or when an identical live document blocks the restore (`meta.reason = "duplicateExists"`). Restoring an already-live document is a no-op success.', + 'Clears the soft-delete tombstone within the 30-day undo grace, and forgets any import-ledger entry for the document\'s source keys. 409 when the tombstone is already purged (the undo window is over, or the document was deleted for good) or when an identical live document blocks the restore (`meta.reason = "duplicateExists"`). Restoring an already-live document is a no-op success.', parameters: [ { name: "id", @@ -1057,6 +1105,62 @@ export const inboundDocumentPaths: NonNullable = { }, }, }, + "/api/documents/inbound/deleted": { + get: { + tags: ["Documents"], + summary: "List Recently deleted", + description: + "The caller's deleted documents that the purge has not removed yet, newest deletion first, metadata only. Each carries `purgeAt`, when the daily purge becomes due for it. Restore one with `POST /api/documents/inbound/{id}/restore` (or the bulk `restore` action); remove one for good ahead of time with `POST /api/documents/inbound/{id}/purge`. Capped at 200 rows (`meta.truncated`, `meta.limit`). Owner-only: refused while acting inside somebody else's record.", + responses: { + "200": { + description: "Recently deleted.", + content: { + "application/json": { + schema: listEnvelope( + deletedDocumentList, + "DeletedDocumentListEnvelope", + ), + }, + }, + }, + ...stdResponses, + }, + }, + }, + "/api/documents/inbound/{id}/purge": { + post: { + tags: ["Documents"], + summary: "Delete a deleted document for good", + description: + "Removes one deleted document now instead of when the grace window ends, through the same purge the daily job runs: the row and everything that cascades from it go, and an imported document's source keys are kept in the import ledger, so an importer re-sending it is still answered as deleted. Only a deleted document can be purged: a live one answers 409 `documents.inbound.notDeleted`. 404 `documents.inbound.notFound` when the caller has no such document. Owner-only.", + parameters: [ + { + name: "id", + in: "path", + required: true, + schema: { type: "string" }, + }, + ], + responses: { + "200": { + description: "Removed for good.", + content: { + "application/json": { + schema: dataEnvelope( + documentPurgeResult, + "DocumentPurgeResultEnvelope", + ), + }, + }, + }, + "409": { + description: "The document is not deleted.", + content: { "application/json": { schema: errorEnvelope } }, + }, + ...stdResponses, + }, + }, + }, "/api/documents/inbound/{id}/original": { get: { tags: ["Documents"], diff --git a/src/lib/query-keys/documents.ts b/src/lib/query-keys/documents.ts index beee7a450..6cbc0168e 100644 --- a/src/lib/query-keys/documents.ts +++ b/src/lib/query-keys/documents.ts @@ -85,6 +85,12 @@ export const documentKeys = { * delete so the quota bar tracks reality. */ inboundDocumentUsage: () => ["documents", "inbound", "usage"] as const, + /** + * Recently deleted (`GET /api/documents/inbound/deleted`). Under the + * `["documents"]` prefix on purpose: a delete, a restore or a purge anywhere + * in the vault repaints it. The literal segment never collides with a cuid. + */ + inboundDocumentsDeleted: () => ["documents", "inbound", "deleted"] as const, /** * Document-scoped AI capability probe * (`GET /api/documents/inbound/capability`). Distinct from the labs diff --git a/src/lib/validations/inbound-documents.ts b/src/lib/validations/inbound-documents.ts index b1adbc621..6ab6137ee 100644 --- a/src/lib/validations/inbound-documents.ts +++ b/src/lib/validations/inbound-documents.ts @@ -878,44 +878,14 @@ export interface DocumentBulkResultDto { error: string | null; } -/** The usage endpoint payload the UI reads before offering an upload. */ -export interface DocumentUsageDto { - usedBytes: number; - quotaBytes: number; - maxFileBytes: number; - acceptedExtensions: string[]; - /** - * Episodes carrying at least one LIVE document link — the vault filter - * bar's condition chips. Sourced server-side so a chip exists even when - * every linked document sits pages deep in the timeline. - */ - linkedEpisodes: DocumentConditionLinkDto[]; - /** - * Procedures and surgeries (visits of kind PROCEDURE) carrying at least one - * LIVE document link — the filter bar's procedure choices, sourced - * server-side for the same reason as `linkedEpisodes`. Empty when the - * caller's grant does not cover the visits section, because the label is - * the visit's own text. - */ - linkedProcedures: DocumentLinkedProcedureDto[]; - /** - * Whether the AI "Suggest details" action can run for this caller — true when - * a provider (vision, or text + local OCR) is configured. The UI hides the - * action when false so it never offers what the endpoint would 422. - */ - assistAvailable: boolean; - /** - * Content-search index state: whether indexing is available to the caller - * (`enabled`, same provider precondition as assist), how many live documents - * are indexed, and the live-document total — so the UI can gauge coverage and - * offer the "index all documents" backfill. - */ - contentIndex: { - enabled: boolean; - indexedCount: number; - totalCount: number; - }; -} +// The usage and Recently deleted payloads live in their own module; kept +// importable from here like every other vault DTO. +export type { + DeletedDocumentDto, + DeletedDocumentListDto, + DocumentPurgeDto, + DocumentUsageDto, +} from "@/lib/documents/vault-read-dtos"; /** Where a document read egresses, vendor-blind. */ export type DocumentEgressClass = "local" | "external"; diff --git a/tests/integration/documents-recently-deleted.test.ts b/tests/integration/documents-recently-deleted.test.ts new file mode 100644 index 000000000..10e474589 --- /dev/null +++ b/tests/integration/documents-recently-deleted.test.ts @@ -0,0 +1,359 @@ +/** + * Recently deleted (#1275) against a real Postgres and the real routes. + * + * 1. The list holds the caller's deleted documents only: not live ones, not + * someone else's; the usage read counts them. + * 2. list → restore → the document is back in the vault list and gone from + * Recently deleted. + * 3. list → delete now → the row is gone, its import source keys (its own + * and an alias) are in the ledger, and an importer re-sending it is still + * answered "deleted"; a live or foreign document cannot be purged. + * 4. A restore removes any ledger entry for the document's keys. + */ +import { beforeEach, describe, expect, it, vi } from "vitest"; +import { NextRequest } from "next/server"; + +process.env.ENCRYPTION_KEY ??= + "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"; + +import { cookieJar, headerJar } from "./mock-next-headers"; +import { getPrismaClient, truncateAllTables } from "./setup"; + +vi.mock("next/headers", async () => { + const { cookieJar, headerJar } = await import("./mock-next-headers"); + return { + headers: vi.fn(async () => ({ + get: (name: string) => headerJar.get(name.toLowerCase()) ?? null, + })), + cookies: vi.fn(async () => ({ + get: (name: string) => { + const value = cookieJar.get(name); + return value ? { name, value } : undefined; + }, + set: (name: string, value: string) => { + cookieJar.set(name, value); + }, + delete: (name: string) => { + cookieJar.delete(name); + }, + })), + }; +}); + +vi.mock("@/lib/db-compat", () => ({ + ensureDbCompatibility: vi.fn().mockResolvedValue(undefined), +})); + +const pdf = (tag: string) => + Buffer.from(`%PDF-1.7\n% ${tag}\n1 0 obj\n<<>>\nendobj\n%%EOF\n`); + +type Ctx = { params: Promise<{ id: string }> }; +const ctx = (id: string): Ctx => ({ params: Promise.resolve({ id }) }); +type Handler = (r: Request, c: Ctx) => Promise; + +async function seedUser(username: string) { + const prisma = getPrismaClient(); + const user = await prisma.user.create({ + data: { + username, + email: `${username}@example.test`, + role: "USER", + timezone: "UTC", + modulePreferencesJson: { inboundDocuments: true }, + }, + }); + const session = await prisma.session.create({ + data: { userId: user.id, expiresAt: new Date(Date.now() + 600_000) }, + }); + return { user, sessionId: session.id }; +} + +function asSession(sessionId: string) { + headerJar.clear(); + cookieJar.clear(); + cookieJar.set("healthlog_session", sessionId); +} + +function upload(bytes: Buffer, fields: Record = {}): Request { + const payload = new Uint8Array(bytes.byteLength); + payload.set(bytes); + const form = new FormData(); + form.append("file", new Blob([payload]), "letter.pdf"); + for (const [k, v] of Object.entries(fields)) form.append(k, v); + const headers: Record = {}; + const auth = headerJar.get("authorization"); + if (auth) headers.authorization = auth; + return new Request("http://localhost/api/documents/inbound", { + method: "POST", + body: form, + headers, + }); +} + +async function routes() { + const list = await import("@/app/api/documents/inbound/route"); + const byId = await import("@/app/api/documents/inbound/[id]/route"); + const restore = + await import("@/app/api/documents/inbound/[id]/restore/route"); + const purge = await import("@/app/api/documents/inbound/[id]/purge/route"); + const deleted = await import("@/app/api/documents/inbound/deleted/route"); + const usage = await import("@/app/api/documents/inbound/usage/route"); + return { + upload: (r: Request) => + (list.POST as unknown as (r: Request) => Promise)(r), + vault: async () => { + const res = await ( + list.GET as unknown as (r: Request) => Promise + )(new Request("http://localhost/api/documents/inbound")); + expect(res.status).toBe(200); + const body = await res.json(); + return (body.data.documents as Array<{ id: string }>).map((d) => d.id); + }, + remove: (id: string) => + (byId.DELETE as unknown as Handler)( + new Request(`http://localhost/api/documents/inbound/${id}`, { + method: "DELETE", + }), + ctx(id), + ), + restore: (id: string) => + (restore.POST as unknown as Handler)( + new NextRequest( + `http://localhost/api/documents/inbound/${id}/restore`, + { method: "POST" }, + ), + ctx(id), + ), + purge: (id: string) => + (purge.POST as unknown as Handler)( + new NextRequest(`http://localhost/api/documents/inbound/${id}/purge`, { + method: "POST", + }), + ctx(id), + ), + deleted: () => + (deleted.GET as unknown as (r: Request) => Promise)( + new Request("http://localhost/api/documents/inbound/deleted"), + ), + usage: async () => { + const res = await ( + usage.GET as unknown as (r: Request) => Promise + )(new Request("http://localhost/api/documents/inbound/usage")); + return (await res.json()).data as { deletedCount: number }; + }, + }; +} + +beforeEach(async () => { + await truncateAllTables(getPrismaClient()); + cookieJar.clear(); + headerJar.clear(); +}); + +describe("Recently deleted — the list", () => { + it("holds the caller's deleted documents only, newest deletion first, with the purge date", async () => { + const r = await routes(); + const other = await seedUser("someone-else"); + asSession(other.sessionId); + const foreign = (await (await r.upload(upload(pdf("theirs")))).json()).data + .id as string; + expect((await r.remove(foreign)).status).toBe(200); + + const { sessionId } = await seedUser("owner"); + asSession(sessionId); + const live = (await (await r.upload(upload(pdf("live")))).json()).data + .id as string; + const first = ( + await ( + await r.upload( + upload(pdf("first"), { + title: "Discharge letter", + kind: "DISCHARGE_LETTER", + documentDate: "2024-05-02", + sourceSystem: "PAPERLESS", + sourceId: "77", + }), + ) + ).json() + ).data.id as string; + const second = (await (await r.upload(upload(pdf("second")))).json()).data + .id as string; + expect((await r.remove(first)).status).toBe(200); + expect((await r.remove(second)).status).toBe(200); + + const res = await r.deleted(); + expect(res.status).toBe(200); + const body = await res.json(); + expect(body.meta).toMatchObject({ truncated: false, limit: 200 }); + expect(body.data.graceDays).toBe(30); + const rows = body.data.documents as Array>; + expect(rows.map((d) => d.id)).toEqual([second, first]); + expect(rows.map((d) => d.id)).not.toContain(live); + expect(rows.map((d) => d.id)).not.toContain(foreign); + const letter = rows[1]; + expect(letter).toMatchObject({ + title: "Discharge letter", + kind: "DISCHARGE_LETTER", + documentDate: "2024-05-02", + sourceSystem: "PAPERLESS", + mimeType: "application/pdf", + }); + expect( + new Date(letter.purgeAt as string).getTime() - + new Date(letter.deletedAt as string).getTime(), + ).toBe(30 * 86_400_000); + // No content, no source id: metadata to recognise the document by. + expect(letter).not.toHaveProperty("contentEncrypted"); + expect(letter).not.toHaveProperty("sourceId"); + + expect((await r.usage()).deletedCount).toBe(2); + }); + + it("list → restore → the document is back in the vault and out of the list", async () => { + const r = await routes(); + const { sessionId } = await seedUser("restorer"); + asSession(sessionId); + const id = (await (await r.upload(upload(pdf("back")))).json()).data + .id as string; + await r.remove(id); + expect(await r.vault()).toEqual([]); + const listed = (await (await r.deleted()).json()).data.documents; + expect(listed.map((d: { id: string }) => d.id)).toEqual([id]); + + expect((await r.restore(id)).status).toBe(200); + expect(await r.vault()).toEqual([id]); + expect((await (await r.deleted()).json()).data.documents).toEqual([]); + expect((await r.usage()).deletedCount).toBe(0); + }); +}); + +describe("Recently deleted — delete now", () => { + it("list → delete now → gone, with its import keys carried into the ledger", async () => { + const prisma = getPrismaClient(); + const r = await routes(); + const { user, sessionId } = await seedUser("purger"); + asSession(sessionId); + const fields = { + sourceSystem: "PAPERLESS", + sourceId: "412", + sourceInstance: "https://paperless.example", + }; + const id = (await (await r.upload(upload(pdf("a"), fields))).json()).data + .id as string; + // The same bytes under a second key: answered with the stored document, + // and the key is remembered on it as an alias. + const alias = await r.upload( + upload(pdf("a"), { ...fields, sourceId: "9" }), + ); + expect((await alias.json()).meta).toMatchObject({ duplicate: true }); + + // A live document cannot be purged in one step. + const live = await r.purge(id); + expect(live.status).toBe(409); + expect((await live.json()).meta.errorCode).toBe( + "documents.inbound.notDeleted", + ); + + await r.remove(id); + const listed = (await (await r.deleted()).json()).data.documents; + expect(listed.map((d: { id: string }) => d.id)).toEqual([id]); + + const purged = await r.purge(id); + expect(purged.status).toBe(200); + expect((await purged.json()).data).toEqual({ id, purged: true }); + + expect(await prisma.inboundDocument.count({ where: { id } })).toBe(0); + expect(await prisma.documentSourceAlias.count()).toBe(0); + expect((await (await r.deleted()).json()).data.documents).toEqual([]); + expect( + await prisma.documentImportKey.findMany({ + where: { userId: user.id }, + select: { sourceSystem: true, sourceId: true, sourceInstance: true }, + orderBy: { sourceId: "asc" }, + }), + ).toEqual([ + { + sourceSystem: "PAPERLESS", + sourceId: "412", + sourceInstance: "https://paperless.example", + }, + { + sourceSystem: "PAPERLESS", + sourceId: "9", + sourceInstance: "https://paperless.example", + }, + ]); + const audit = await prisma.auditLog.findFirst({ + where: { userId: user.id, action: "documents.inbound.purge" }, + }); + expect(audit).not.toBeNull(); + + // An importer re-sending either key is still answered "deleted", and + // nothing is stored. + for (const sourceId of ["412", "9"]) { + const again = await r.upload(upload(pdf("a"), { ...fields, sourceId })); + expect(again.status).toBe(200); + expect((await again.json()).meta).toEqual({ + duplicate: true, + deleted: true, + }); + } + expect(await prisma.inboundDocument.count()).toBe(0); + + // Restoring it now is over. + const gone = await r.restore(id); + expect(gone.status).toBe(409); + expect((await gone.json()).meta.reason).toBe("purged"); + }); + + it("answers 404 for someone else's deleted document and leaves it alone", async () => { + const prisma = getPrismaClient(); + const r = await routes(); + const owner = await seedUser("owner-b"); + asSession(owner.sessionId); + const id = (await (await r.upload(upload(pdf("mine")))).json()).data + .id as string; + await r.remove(id); + + const intruder = await seedUser("intruder"); + asSession(intruder.sessionId); + const res = await r.purge(id); + expect(res.status).toBe(404); + expect(await prisma.inboundDocument.count({ where: { id } })).toBe(1); + expect((await (await r.deleted()).json()).data.documents).toEqual([]); + }); +}); + +describe("Recently deleted — restore and the import ledger", () => { + it("a restore removes the ledger entries for the document's own and alias keys", async () => { + const prisma = getPrismaClient(); + const r = await routes(); + const { user, sessionId } = await seedUser("ledger"); + asSession(sessionId); + const fields = { sourceSystem: "PAPRA", sourceId: "doc_1" }; + const id = (await (await r.upload(upload(pdf("k"), fields))).json()).data + .id as string; + await r.upload(upload(pdf("k"), { ...fields, sourceId: "doc_2" })); + await r.remove(id); + + // A ledger entry for each key, as a purge elsewhere (or an older run) + // would have left; one for an unrelated key stays. + await prisma.documentImportKey.createMany({ + data: [ + { userId: user.id, sourceSystem: "PAPRA", sourceId: "doc_1" }, + { userId: user.id, sourceSystem: "PAPRA", sourceId: "doc_2" }, + { userId: user.id, sourceSystem: "PAPRA", sourceId: "unrelated" }, + ], + }); + + expect((await r.restore(id)).status).toBe(200); + expect( + ( + await prisma.documentImportKey.findMany({ + where: { userId: user.id }, + select: { sourceId: true }, + }) + ).map((k) => k.sourceId), + ).toEqual(["unrelated"]); + }); +}); From 52fa5a06b1d606fed27000ab2ac357ae31e37f1e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Sat, 24 Oct 2026 20:55:19 +0200 Subject: [PATCH 11/11] feat(documents): let the picker bring back a deleted document The document picker showed a document deleted in HealthLog as "Deleted in HealthLog" and would not let it be ticked, so a person who deleted an imported document by mistake met a dead end exactly where they would look for it again. Such a row can now be ticked, and the import sends revive: true for it. A document still in Recently deleted is restored through the shared restore, nothing is fetched, and the row reads "Restored from Recently deleted". One the purge already removed is fetched and stored again, and only once it is stored does the ledger forget its key, so a failed fetch leaves the deletion standing. Without revive the route answers "deleted" as before, and the token upload has no such switch: a Paperless workflow or the import script re-sending a deleted document is still refused, whether it is in Recently deleted or long purged. The search names the document a deleted key still points at (documentId, null once purged), so the row can say "In Recently deleted" rather than "Deleted in HealthLog". The outcome gains "restored", sent only in answer to revive. The row logic moves to picker-rows.ts. Refs #1275 --- docs/api/openapi.yaml | 19 ++- docs/self-hosting/document-sources.md | 19 ++- messages/de.json | 5 +- messages/en.json | 5 +- messages/es.json | 5 +- messages/fr.json | 5 +- messages/it.json | 5 +- messages/ko.json | 5 +- messages/nl.json | 5 +- messages/pl.json | 5 +- messages/pt-BR.json | 5 +- .../baselines/source-function-lines.json | 4 +- .../sources/[system]/import/route.ts | 150 ++++++++++++++++-- .../sources/[system]/search/route.ts | 4 +- .../sources/__tests__/picker-rows.test.ts | 85 ++++++++++ .../sources/document-source-picker.tsx | 86 ++++------ .../documents/sources/picker-rows.ts | 91 +++++++++++ src/lib/documents/source-key.ts | 37 ++++- src/lib/documents/sources/types.ts | 13 +- src/lib/openapi/routes/document-sources.ts | 17 +- src/lib/validations/document-sources.ts | 8 + tests/integration/document-sources.test.ts | 87 +++++++++- 22 files changed, 562 insertions(+), 103 deletions(-) create mode 100644 src/components/documents/sources/__tests__/picker-rows.test.ts create mode 100644 src/components/documents/sources/picker-rows.ts diff --git a/docs/api/openapi.yaml b/docs/api/openapi.yaml index cf89cde46..b92152575 100644 --- a/docs/api/openapi.yaml +++ b/docs/api/openapi.yaml @@ -25487,9 +25487,12 @@ paths: waits and then answers `duplicate`. The key check counts toward the lookup allowance, the fetch toward the search allowance, and only a file that arrived toward the upload allowance. A source key the vault already holds is answered without a download: `duplicate` for a live document (same id), `deleted` for one the person deleted - in HealthLog, which is never stored again. Bytes already in the vault under another key are `duplicate` too, and - the key is remembered for that document. `link` files the document against the record the picker was opened from - (a condition, a visit or a vaccination, which must be the caller's own and live; checked before anything is + in HealthLog, which is not stored again unless the request sends `revive: true`: then a document still in + Recently deleted is restored (`restored`), and one already removed for good is fetched and stored again, its key + leaving the import ledger. A token upload (`POST /api/documents/inbound`) has no such switch, so an importer + never brings a deleted document back. Bytes already in the vault under another key are `duplicate` too, and the + key is remembered for that document. `link` files the document against the record the picker was opened from (a + condition, a visit or a vaccination, which must be the caller's own and live; checked before anything is fetched), including a duplicate. `kind` sets the document type (default OTHER). Automatic AI reading follows the person's own setting. Counts against the person's upload allowance (60 an hour) and storage quota. 201 for a new document, 200 otherwise. Cookie session only; not delegable." @@ -25537,11 +25540,14 @@ paths: required: - kind - id + revive: + type: boolean required: - sourceId responses: "200": - description: Already held (`duplicate`) or deleted in HealthLog (`deleted`). + description: Already held (`duplicate`), deleted in HealthLog (`deleted`), or brought back from Recently deleted + (`restored`). content: application/json: schema: @@ -90451,6 +90457,8 @@ components: - imported - deleted documentId: + description: The vault document for `imported`. For `deleted`, the document while it is still in Recently deleted + (restorable), null once it has been removed for good. type: - string - "null" @@ -90489,6 +90497,9 @@ components: - imported - duplicate - deleted + - restored + description: "`restored` only answers a request that sent `revive: true`: the deleted document was still in Recently + deleted and is back in the vault." documentId: type: - string diff --git a/docs/self-hosting/document-sources.md b/docs/self-hosting/document-sources.md index 4a30cdd2c..e0be3e5cc 100644 --- a/docs/self-hosting/document-sources.md +++ b/docs/self-hosting/document-sources.md @@ -82,13 +82,18 @@ limit for a document). - Imports count against the person's normal upload allowance (60 an hour) and storage quota. -- A document that is already in HealthLog is not stored twice, and one the - person deleted in HealthLog is not brought back by picking it again. Each - import is keyed by system, instance (the archive's origin) and document id, - so the same id on two Paperless-ngx instances is two documents; a key - imported before v1.39.3 carries no instance and matches any instance. These - are the same rules the document token and `scripts/import-documents.mjs` - follow. +- A document that is already in HealthLog is not stored twice. Each import is + keyed by system, instance (the archive's origin) and document id, so the + same id on two Paperless-ngx instances is two documents; a key imported + before v1.39.3 carries no instance and matches any instance. These are the + same rules the document token and `scripts/import-documents.mjs` follow. +- A document the person deleted in HealthLog is never brought back by the + document token or the import script, however often they re-send it. The + person can bring it back themselves: from Recently deleted on the Documents + page for 30 days, or by picking it again in the picker, which says "In Recently + deleted" or "Deleted in HealthLog" on such a row. Picking one still in + Recently deleted restores that same document; picking one already removed + for good imports it again. - A picked document follows the person's own automatic AI reading setting. - Only the owner of a record can connect an archive or import from it. People with shared access to a record, whatever their level, do not see the picker. diff --git a/messages/de.json b/messages/de.json index 4ef2f6ce8..bd07d9f7a 100644 --- a/messages/de.json +++ b/messages/de.json @@ -11890,6 +11890,7 @@ "loadMore": "Mehr laden", "stateImported": "In HealthLog", "stateDeleted": "In HealthLog gelöscht", + "stateInTrash": "Unter Zuletzt gelöscht", "selectedCount": "{count} von höchstens {max} ausgewählt", "importAction": "{count} importieren", "importing": "Import läuft, ein Dokument nach dem anderen …", @@ -11903,7 +11904,9 @@ "importedLinked": "Importiert und verknüpft", "duplicate": "Schon in HealthLog", "duplicateLinked": "Schon in HealthLog, jetzt verknüpft", - "deleted": "In HealthLog gelöscht, bleibt gelöscht" + "deleted": "In HealthLog gelöscht, bleibt gelöscht", + "restored": "Aus Zuletzt gelöscht wiederhergestellt", + "restoredLinked": "Aus Zuletzt gelöscht wiederhergestellt und verknüpft" }, "errors": { "unavailable": "Dokumentarchive sind auf diesem Server nicht freigeschaltet.", diff --git a/messages/en.json b/messages/en.json index 71c85501f..da7365a1e 100644 --- a/messages/en.json +++ b/messages/en.json @@ -11890,6 +11890,7 @@ "loadMore": "Load more", "stateImported": "In HealthLog", "stateDeleted": "Deleted in HealthLog", + "stateInTrash": "In Recently deleted", "selectedCount": "{count} of up to {max} selected", "importAction": "Import {count}", "importing": "Importing, one document at a time…", @@ -11903,7 +11904,9 @@ "importedLinked": "Imported and linked", "duplicate": "Already in HealthLog", "duplicateLinked": "Already in HealthLog, now linked", - "deleted": "Deleted in HealthLog, left alone" + "deleted": "Deleted in HealthLog, left alone", + "restored": "Restored from Recently deleted", + "restoredLinked": "Restored from Recently deleted and linked" }, "errors": { "unavailable": "Document archives are not enabled on this server.", diff --git a/messages/es.json b/messages/es.json index a90623fc8..a87864f1c 100644 --- a/messages/es.json +++ b/messages/es.json @@ -11890,6 +11890,7 @@ "loadMore": "Cargar más", "stateImported": "En HealthLog", "stateDeleted": "Eliminado en HealthLog", + "stateInTrash": "En Eliminados recientemente", "selectedCount": "{count} de un máximo de {max} seleccionados", "importAction": "Importar {count}", "importing": "Importando, un documento tras otro…", @@ -11903,7 +11904,9 @@ "importedLinked": "Importado y vinculado", "duplicate": "Ya está en HealthLog", "duplicateLinked": "Ya está en HealthLog, ahora vinculado", - "deleted": "Eliminado en HealthLog, se deja así" + "deleted": "Eliminado en HealthLog, se deja así", + "restored": "Restaurado desde Eliminados recientemente", + "restoredLinked": "Restaurado desde Eliminados recientemente y vinculado" }, "errors": { "unavailable": "Los archivos de documentos no están habilitados en este servidor.", diff --git a/messages/fr.json b/messages/fr.json index 7d93dc773..61afc2d54 100644 --- a/messages/fr.json +++ b/messages/fr.json @@ -11890,6 +11890,7 @@ "loadMore": "Charger plus", "stateImported": "Dans HealthLog", "stateDeleted": "Supprimé dans HealthLog", + "stateInTrash": "Dans Supprimés récemment", "selectedCount": "{count} sélectionnés sur {max} au plus", "importAction": "Importer {count}", "importing": "Import en cours, un document après l'autre…", @@ -11903,7 +11904,9 @@ "importedLinked": "Importé et lié", "duplicate": "Déjà dans HealthLog", "duplicateLinked": "Déjà dans HealthLog, désormais lié", - "deleted": "Supprimé dans HealthLog, laissé tel quel" + "deleted": "Supprimé dans HealthLog, laissé tel quel", + "restored": "Restauré depuis Supprimés récemment", + "restoredLinked": "Restauré depuis Supprimés récemment et lié" }, "errors": { "unavailable": "Les archives de documents ne sont pas activées sur ce serveur.", diff --git a/messages/it.json b/messages/it.json index 1e6f6cce6..c1affc89d 100644 --- a/messages/it.json +++ b/messages/it.json @@ -11890,6 +11890,7 @@ "loadMore": "Carica altri", "stateImported": "Già in HealthLog", "stateDeleted": "Eliminato in HealthLog", + "stateInTrash": "In Eliminati di recente", "selectedCount": "{count} selezionati su un massimo di {max}", "importAction": "Importa {count}", "importing": "Importazione in corso, un documento alla volta…", @@ -11903,7 +11904,9 @@ "importedLinked": "Importato e collegato", "duplicate": "Già in HealthLog", "duplicateLinked": "Già in HealthLog, ora collegato", - "deleted": "Eliminato in HealthLog, lasciato così" + "deleted": "Eliminato in HealthLog, lasciato così", + "restored": "Ripristinato da Eliminati di recente", + "restoredLinked": "Ripristinato da Eliminati di recente e collegato" }, "errors": { "unavailable": "Gli archivi di documenti non sono abilitati su questo server.", diff --git a/messages/ko.json b/messages/ko.json index dc0f9212b..7c9a30697 100644 --- a/messages/ko.json +++ b/messages/ko.json @@ -11890,6 +11890,7 @@ "loadMore": "더 불러오기", "stateImported": "HealthLog에 있음", "stateDeleted": "HealthLog에서 삭제됨", + "stateInTrash": "최근 삭제됨에 있음", "selectedCount": "최대 {max}개 중 {count}개 선택", "importAction": "{count}개 가져오기", "importing": "문서를 하나씩 가져오는 중…", @@ -11903,7 +11904,9 @@ "importedLinked": "가져오고 연결함", "duplicate": "이미 HealthLog에 있음", "duplicateLinked": "이미 HealthLog에 있음, 이제 연결됨", - "deleted": "HealthLog에서 삭제됨, 그대로 둠" + "deleted": "HealthLog에서 삭제됨, 그대로 둠", + "restored": "최근 삭제됨에서 복구했어요", + "restoredLinked": "최근 삭제됨에서 복구하고 연결했어요" }, "errors": { "unavailable": "이 서버에서는 문서 보관함이 활성화되어 있지 않습니다.", diff --git a/messages/nl.json b/messages/nl.json index a831c0207..4441b4dd8 100644 --- a/messages/nl.json +++ b/messages/nl.json @@ -11890,6 +11890,7 @@ "loadMore": "Meer laden", "stateImported": "Staat in HealthLog", "stateDeleted": "Verwijderd in HealthLog", + "stateInTrash": "In Recent verwijderd", "selectedCount": "{count} van maximaal {max} geselecteerd", "importAction": "{count} importeren", "importing": "Importeren, één document tegelijk…", @@ -11903,7 +11904,9 @@ "importedLinked": "Geïmporteerd en gekoppeld", "duplicate": "Staat al in HealthLog", "duplicateLinked": "Staat al in HealthLog, nu gekoppeld", - "deleted": "Verwijderd in HealthLog, ongemoeid gelaten" + "deleted": "Verwijderd in HealthLog, ongemoeid gelaten", + "restored": "Hersteld uit Recent verwijderd", + "restoredLinked": "Hersteld uit Recent verwijderd en gekoppeld" }, "errors": { "unavailable": "Documentarchieven zijn niet ingeschakeld op deze server.", diff --git a/messages/pl.json b/messages/pl.json index 679c7ee7d..8c84eafc1 100644 --- a/messages/pl.json +++ b/messages/pl.json @@ -11890,6 +11890,7 @@ "loadMore": "Wczytaj więcej", "stateImported": "W HealthLog", "stateDeleted": "Usunięty w HealthLog", + "stateInTrash": "W sekcji Ostatnio usunięte", "selectedCount": "Wybrano {count} z maksymalnie {max}", "importAction": "Importuj ({count})", "importing": "Importowanie, dokument po dokumencie…", @@ -11903,7 +11904,9 @@ "importedLinked": "Zaimportowano i powiązano", "duplicate": "Już jest w HealthLog", "duplicateLinked": "Już jest w HealthLog, teraz powiązany", - "deleted": "Usunięty w HealthLog, pozostaje usunięty" + "deleted": "Usunięty w HealthLog, pozostaje usunięty", + "restored": "Przywrócono z sekcji Ostatnio usunięte", + "restoredLinked": "Przywrócono z sekcji Ostatnio usunięte i powiązano" }, "errors": { "unavailable": "Archiwa dokumentów nie są włączone na tym serwerze.", diff --git a/messages/pt-BR.json b/messages/pt-BR.json index d19fdec09..d325f0552 100644 --- a/messages/pt-BR.json +++ b/messages/pt-BR.json @@ -11890,6 +11890,7 @@ "loadMore": "Carregar mais", "stateImported": "No HealthLog", "stateDeleted": "Excluído no HealthLog", + "stateInTrash": "Em Excluídos recentemente", "selectedCount": "{count} de até {max} selecionados", "importAction": "Importar {count}", "importing": "Importando, um documento por vez…", @@ -11903,7 +11904,9 @@ "importedLinked": "Importado e vinculado", "duplicate": "Já no HealthLog", "duplicateLinked": "Já no HealthLog, agora vinculado", - "deleted": "Excluído no HealthLog, deixado como está" + "deleted": "Excluído no HealthLog, deixado como está", + "restored": "Restaurado de Excluídos recentemente", + "restoredLinked": "Restaurado de Excluídos recentemente e vinculado" }, "errors": { "unavailable": "Arquivos de documentos não estão habilitados neste servidor.", diff --git a/src/__tests__/baselines/source-function-lines.json b/src/__tests__/baselines/source-function-lines.json index 7b0728781..3a8b2011a 100644 --- a/src/__tests__/baselines/source-function-lines.json +++ b/src/__tests__/baselines/source-function-lines.json @@ -1,7 +1,7 @@ { "$comment": "Production functions over 200 lines, keyed \"file :: name\", and their length. Maintained by src/__tests__/source-size-ratchet.test.ts.", "count": 351, - "total": 113742, + "total": 113740, "entries": { "app/achievements/page.tsx :: AchievementsPage": 206, "app/api/admin/backups/upload/route.ts :: apiHandler(arg)": 314, @@ -100,7 +100,7 @@ "components/documents/document-record-links.tsx :: DocumentRecordLinks": 202, "components/documents/document-timeline.tsx :: DocumentTimeline": 314, "components/documents/documents-view.tsx :: DocumentsView": 303, - "components/documents/sources/document-source-picker.tsx :: DocumentSourcePicker": 565, + "components/documents/sources/document-source-picker.tsx :: DocumentSourcePicker": 563, "components/illness/log-day-sheet.tsx :: LogDaySheet": 265, "components/illness/new-episode-sheet.tsx :: NewEpisodeSheet": 264, "components/insights/coach-panel/attachment-picker.tsx :: AttachmentPicker": 208, diff --git a/src/app/api/documents/sources/[system]/import/route.ts b/src/app/api/documents/sources/[system]/import/route.ts index a5f4cb90f..854180ac0 100644 --- a/src/app/api/documents/sources/[system]/import/route.ts +++ b/src/app/api/documents/sources/[system]/import/route.ts @@ -11,7 +11,13 @@ * 2. the source key answers first, without a download: a document already * imported (or already holding this key as an alias) is a `duplicate`, * and one the person deleted here is `deleted` — nothing is fetched and - * nothing is stored, exactly as the token upload answers the same key; + * nothing is stored, exactly as the token upload answers the same key. + * Unless the person asked to bring it back (`revive`, sent only for a + * row they picked knowing it was deleted): a document still in Recently + * deleted is restored (`restored`, nothing fetched), and one the purge + * already removed is fetched and stored like a new one, after which the + * import ledger forgets its key. The token upload has no such switch, so + * a deletion keeps holding against every automatic re-send; * 3. only then is the source asked (on the search allowance, like every * request to it) and an upload slot taken. One import per key and person * runs at a time under an advisory lock, so a double click downloads the @@ -37,6 +43,7 @@ import { NextRequest, NextResponse } from "next/server"; import { apiHandler, requireAuth } from "@/lib/api-handler"; import { apiError, getClientIp } from "@/lib/api-response"; +import { auditLog } from "@/lib/auth/audit"; import { parseJsonBody } from "@/lib/api/body"; import { getAiCapability } from "@/lib/ai/capabilities/gate"; import { prisma } from "@/lib/db"; @@ -49,6 +56,7 @@ import { import { checkSourceLookupRateLimit, findSourceKey, + forgetDeletedSourceKey, type SourceKeyMatch, } from "@/lib/documents/source-key"; import type { SourceClient } from "@/lib/documents/sources/clients"; @@ -71,6 +79,7 @@ import { acquireDocumentUploadSlot, resolveDocumentLimits, } from "@/lib/documents/upload-policy"; +import { restoreDocumentTombstone } from "@/lib/documents/tombstones"; import { linkTargets } from "@/lib/links"; import { annotate } from "@/lib/logging/context"; import { checkRateLimit, rateLimitHeaders } from "@/lib/rate-limit"; @@ -148,18 +157,82 @@ function respond( ); } -/** A key the vault already holds: link a live one, never revive a deleted one. */ +/** The key the request names, for the ledger. */ +interface PickedKey { + system: DocumentPickerSystem; + sourceId: string; + instance: string; +} + +/** + * A key the vault already holds: link a live one; leave a deleted one alone + * unless the person asked to bring it back (`revive`). Then a document still + * in Recently deleted is restored here, and for one the purge already removed + * this answers null: the caller fetches and stores it again. + */ async function answerHeldKey( - system: string, + request: Request, + key: PickedKey, userId: string, link: LinkTarget | undefined, match: SourceKeyMatch, -): Promise { + revive: boolean, +): Promise { if (match.state === "deleted") { - return respond(system, "deleted", match.id, false); + if (!revive) return respond(key.system, "deleted", match.id, false); + if (match.id === null) return null; + return restoreFromPicker(request, key, userId, link, match.id); } const linked = await linkDocument(userId, link, match.document.id); - return respond(system, "duplicate", match.document.id, linked); + return respond(key.system, "duplicate", match.document.id, linked); +} + +/** + * Bring a picked document back out of Recently deleted, through the same + * restore the vault uses (which also clears the ledger for the document's own + * keys), and forget the picked key's ledger entry the way a re-import would. + */ +async function restoreFromPicker( + request: Request, + key: PickedKey, + userId: string, + link: LinkTarget | undefined, + documentId: string, +): Promise { + const outcome = await restoreDocumentTombstone(userId, documentId); + switch (outcome.kind) { + case "gone": + // Purged between the key check and the restore: nothing to bring back + // from Recently deleted, and nothing is fetched on this request. + return respond(key.system, "deleted", null, false); + case "duplicate": { + // The same bytes are live under another document: that one is the + // person's copy, so the pick links it, like any duplicate. + const existing = outcome.existingId; + const linked = existing + ? await linkDocument(userId, link, existing) + : false; + return respond(key.system, "duplicate", existing, linked); + } + case "live": + case "restored": { + await forgetDeletedSourceKey( + userId, + key.system, + key.sourceId, + key.instance, + ); + if (outcome.kind === "restored") { + await auditLog("documents.inbound.restore", { + userId, + ipAddress: getClientIp(request), + details: { documentId, via: "documentPicker" }, + }); + } + const linked = await linkDocument(userId, link, documentId); + return respond(key.system, "restored", documentId, linked); + } + } } export const POST = apiHandler( @@ -177,6 +250,7 @@ export const POST = apiHandler( }); if (!parsed.ok) return parsed.error; const { sourceId, kind, link } = parsed.data; + const revive = parsed.data.revive === true; // Only an id this system could have issued goes into a request path. if (!isSourceDocumentId(system, sourceId)) { return apiError("Not a document id of this archive.", 422); @@ -210,8 +284,19 @@ export const POST = apiHandler( rateLimitHeaders(lookupRl), ); } + const key: PickedKey = { system, sourceId, instance }; const held = await findSourceKey(userId, system, sourceId, instance); - if (held) return answerHeldKey(system, userId, link, held); + if (held) { + const answered = await answerHeldKey( + request, + key, + userId, + link, + held, + revive, + ); + if (answered) return answered; + } // A request to the source counts against the search allowance like any // other; the upload allowance is charged only once a file arrived. @@ -249,7 +334,17 @@ export const POST = apiHandler( FROM pg_advisory_xact_lock(hashtextextended(${`documents-import:${userId}:${system}:${instance}:${sourceId}`}, 0)) `; const raced = await findSourceKey(userId, system, sourceId, instance); - if (raced) return answerHeldKey(system, userId, link, raced); + if (raced) { + const answered = await answerHeldKey( + request, + key, + userId, + link, + raced, + revive, + ); + if (answered) return answered; + } return importOne({ request, client, @@ -259,6 +354,7 @@ export const POST = apiHandler( sourceId, kind: kind ?? null, link, + revive, }); }, { maxWait: IMPORT_LOCK_WAIT_MS, timeout: IMPORT_LOCK_HOLD_MS }, @@ -279,9 +375,21 @@ async function importOne(args: { sourceId: string; kind: InboundDocumentKindValue | null; link: LinkTarget | undefined; + /** The person asked to bring back a document they deleted here. */ + revive: boolean; }): Promise { - const { request, client, userId, system, instance, sourceId, kind, link } = - args; + const { + request, + client, + userId, + system, + instance, + sourceId, + kind, + link, + revive, + } = args; + const key: PickedKey = { system, sourceId, instance }; const limits = await resolveDocumentLimits(userId); let meta; let file; @@ -326,16 +434,36 @@ async function importOne(args: { documentAi: () => getAiCapability("documentAi"), }); + // A key the purge removed was kept in the ledger; once the person brought + // the document back by hand, the ledger stops saying it is deleted. Done + // only after the document is in the vault again, so a fetch that failed + // leaves the deletion standing. + const forgetIfRevived = async () => { + if (revive) + await forgetDeletedSourceKey(userId, system, sourceId, instance); + }; + switch (result.kind) { case "sourceKey": - return answerHeldKey(system, userId, link, result.match); + return ( + (await answerHeldKey( + request, + key, + userId, + link, + result.match, + revive, + )) ?? respond(system, "deleted", null, false) + ); case "duplicate": { // The same bytes were already in the vault (an upload, or another // archive); the key is now remembered for that document. + await forgetIfRevived(); const linked = await linkDocument(userId, link, result.document.id); return respond(system, "duplicate", result.document.id, linked); } case "stored": { + await forgetIfRevived(); const linked = await linkDocument(userId, link, result.document.id); return respond(system, "imported", result.document.id, linked); } diff --git a/src/app/api/documents/sources/[system]/search/route.ts b/src/app/api/documents/sources/[system]/search/route.ts index edaf7a9c3..22e4138b4 100644 --- a/src/app/api/documents/sources/[system]/search/route.ts +++ b/src/app/api/documents/sources/[system]/search/route.ts @@ -132,7 +132,9 @@ export const GET = apiHandler( tags: item.tags, sizeBytes: item.sizeBytes, state: held?.state ?? "new", - documentId: held?.state === "imported" ? held.documentId : null, + // The vault document for `imported`; for `deleted`, the one still in + // Recently deleted (null once the purge removed it). + documentId: held?.documentId ?? null, }; }); diff --git a/src/components/documents/sources/__tests__/picker-rows.test.ts b/src/components/documents/sources/__tests__/picker-rows.test.ts new file mode 100644 index 000000000..ac3d310b2 --- /dev/null +++ b/src/components/documents/sources/__tests__/picker-rows.test.ts @@ -0,0 +1,85 @@ +import { describe, expect, it } from "vitest"; + +import type { DocumentSourceResultDto } from "@/lib/documents/sources/types"; + +import { + deletedSourceIds, + heldBadgeLabel, + isPickable, + runLabel, + runOutcome, +} from "../picker-rows"; + +function row( + over: Partial = {}, +): DocumentSourceResultDto { + return { + sourceId: "41", + title: "Blood test", + date: null, + tags: [], + sizeBytes: null, + state: "new", + documentId: null, + ...over, + }; +} + +const t = (key: string) => key; + +describe("isPickable", () => { + it("ticks a new row and a deleted one (asking for it back)", () => { + expect(isPickable(row(), false)).toBe(true); + expect(isPickable(row({ state: "deleted", documentId: "d1" }), false)).toBe( + true, + ); + expect(isPickable(row({ state: "deleted" }), false)).toBe(true); + }); + + it("ticks an imported row only when the pick links it or hands it on", () => { + const imported = row({ state: "imported", documentId: "d1" }); + expect(isPickable(imported, false)).toBe(false); + expect(isPickable(imported, true)).toBe(true); + }); +}); + +describe("deletedSourceIds", () => { + it("names exactly the rows held as deleted, which the import revives", () => { + const ids = deletedSourceIds([ + row({ sourceId: "1" }), + row({ sourceId: "2", state: "deleted", documentId: "d2" }), + row({ sourceId: "3", state: "imported", documentId: "d3" }), + row({ sourceId: "4", state: "deleted" }), + ]); + expect([...ids].sort()).toEqual(["2", "4"]); + }); +}); + +describe("heldBadgeLabel", () => { + it("tells a document still in Recently deleted from one removed for good", () => { + expect(heldBadgeLabel(t, row({ state: "deleted", documentId: "d1" }))).toBe( + "documents.sourcePicker.stateInTrash", + ); + expect(heldBadgeLabel(t, row({ state: "deleted" }))).toBe( + "documents.sourcePicker.stateDeleted", + ); + expect(heldBadgeLabel(t, row({ state: "imported", documentId: "d" }))).toBe( + "documents.sourcePicker.stateImported", + ); + }); +}); + +describe("a restored row", () => { + it("reads as restored, linked or not", () => { + expect(runLabel(t, { state: "restored", linked: false })).toBe( + "documents.sourcePicker.result.restored", + ); + expect(runLabel(t, { state: "restored", linked: true })).toBe( + "documents.sourcePicker.result.restoredLinked", + ); + }); + + it("counts toward a successful run", () => { + expect(runOutcome({ imported: 1, present: 0, failed: 0 })).toBe("success"); + }); +}); diff --git a/src/components/documents/sources/document-source-picker.tsx b/src/components/documents/sources/document-source-picker.tsx index c492f5607..7b9732d7e 100644 --- a/src/components/documents/sources/document-source-picker.tsx +++ b/src/components/documents/sources/document-source-picker.tsx @@ -9,7 +9,11 @@ * field opens focused and debounces into the source's own name search. Tag and * date are filters beside it, not the entry point. Each result says what * HealthLog already holds under that source key: already imported, deleted - * here (not offered again: a deletion was a decision), or new. + * here, or new. A deleted one can be ticked too, and that is the person + * deciding again: one still in Recently deleted is restored (nothing is + * fetched), one already removed for good is imported afresh. The import sends + * `revive` only for those rows; an importer's upload never can, so a deletion + * still holds against every automatic re-send. * * The import runs one document per request, in order, so every row can show * its own progress and result, and one failure does not hide the others. A @@ -59,7 +63,6 @@ import { apiGet, apiPost } from "@/lib/api/api-fetch"; import { DOCUMENT_PICKER_MAX_SELECTION, slugForSystem, - type DocumentImportOutcome, type DocumentPickerLinkKind, type DocumentPickerSystem, type DocumentSourceImportDto, @@ -73,7 +76,6 @@ import { cn } from "@/lib/utils"; import type { InboundDocumentKindValue } from "@/lib/validations/inbound-documents"; import { WrittenOutcomeLine } from "@/components/outcome/written-outcome-line"; -import type { WrittenOutcome } from "@/lib/outcome/written-outcome"; import { formatBytes } from "@/lib/format/bytes"; import { @@ -84,6 +86,14 @@ import { import { Segmented } from "@/components/ui/segmented"; import { systemName } from "./use-document-sources"; import { InlineAction } from "@/components/ui/inline-action"; +import { + deletedSourceIds, + heldBadgeLabel, + isPickable, + runLabel, + runOutcome, + type RowRun, +} from "./picker-rows"; export interface DocumentPickerLink { kind: DocumentPickerLinkKind; @@ -103,12 +113,6 @@ export interface DocumentSourcePickerProps { onImported?: (documentIds: string[]) => void; } -type RowRun = - | { state: "waiting" } - | { state: "working" } - | { state: DocumentImportOutcome; linked: boolean } - | { state: "failed"; message: string }; - const SEARCH_DEBOUNCE_MS = 300; const IMPORT_TIMEOUT_MS = 120_000; @@ -242,11 +246,9 @@ export function DocumentSourcePicker({ return out; }, [search.data]); - // An already imported document is worth picking only when the pick does - // something with it: link it, or hand it to the form. const canTakeImported = Boolean(link || onImported); const selectable = (row: DocumentSourceResultDto) => - row.state === "new" || (row.state === "imported" && canTakeImported); + isPickable(row, canTakeImported); const toggle = (row: DocumentSourceResultDto) => { if (running || finished || !selectable(row)) return; @@ -269,6 +271,7 @@ export function DocumentSourcePicker({ setRunning(true); setStopMessage(null); const order = [...selected.keys()]; + const revive = deletedSourceIds(results); setRuns(new Map(order.map((id) => [id, { state: "waiting" }]))); const handed: string[] = []; const mark = (id: string, run: RowRun) => { @@ -287,6 +290,7 @@ export function DocumentSourcePicker({ sourceId, ...(kind ? { kind } : {}), ...(link ? { link } : {}), + ...(revive.has(sourceId) ? { revive: true } : {}), }, // The server may take up to a minute to fetch a large file from // the source; the default 15 s client timeout would abandon it. @@ -324,7 +328,7 @@ export function DocumentSourcePicker({ let present = 0; let failed = 0; for (const run of runs.values()) { - if (run.state === "imported") imported += 1; + if (run.state === "imported" || run.state === "restored") imported += 1; else if (run.state === "duplicate" || run.state === "deleted") present += 1; else if (run.state === "failed") failed += 1; @@ -643,9 +647,7 @@ export function DocumentSourcePicker({ {row.state !== "new" && !run ? ( - {row.state === "imported" - ? t("documents.sourcePicker.stateImported") - : t("documents.sourcePicker.stateDeleted")} + {heldBadgeLabel(t, row)} ) : null} @@ -678,41 +680,6 @@ export function DocumentSourcePicker({ ); } -/** How the run went, in the shared outcome vocabulary. */ -function runOutcome(counts: { - imported: number; - present: number; - failed: number; -}): WrittenOutcome { - const held = counts.imported + counts.present; - if (counts.failed === 0) return held > 0 ? "success" : "empty"; - return held > 0 ? "partial" : "failed"; -} - -/** A row's result in words, or null while it has none yet. */ -function runLabel( - t: (key: string, params?: Record) => string, - run: RowRun, -): string | null { - switch (run.state) { - case "waiting": - case "working": - return null; - case "failed": - return run.message; - case "deleted": - return t("documents.sourcePicker.result.deleted"); - case "imported": - return run.linked - ? t("documents.sourcePicker.result.importedLinked") - : t("documents.sourcePicker.result.imported"); - case "duplicate": - return run.linked - ? t("documents.sourcePicker.result.duplicateLinked") - : t("documents.sourcePicker.result.duplicate"); - } -} - /** One row's progress or result, under its title. */ function RunLine({ run }: { run: RowRun }) { const { t } = useTranslations(); @@ -751,19 +718,24 @@ function RunLine({ run }: { run: RowRun }) { ); case "imported": case "duplicate": + case "restored": return ( - {run.state === "imported" + {run.state === "restored" ? run.linked - ? t("documents.sourcePicker.result.importedLinked") - : t("documents.sourcePicker.result.imported") - : run.linked - ? t("documents.sourcePicker.result.duplicateLinked") - : t("documents.sourcePicker.result.duplicate")} + ? t("documents.sourcePicker.result.restoredLinked") + : t("documents.sourcePicker.result.restored") + : run.state === "imported" + ? run.linked + ? t("documents.sourcePicker.result.importedLinked") + : t("documents.sourcePicker.result.imported") + : run.linked + ? t("documents.sourcePicker.result.duplicateLinked") + : t("documents.sourcePicker.result.duplicate")} ); } diff --git a/src/components/documents/sources/picker-rows.ts b/src/components/documents/sources/picker-rows.ts new file mode 100644 index 000000000..eb06e24f8 --- /dev/null +++ b/src/components/documents/sources/picker-rows.ts @@ -0,0 +1,91 @@ +/** + * The document picker's row logic, apart from its rendering: which rows can be + * ticked, which ask for a deleted document back, what a held row's badge and + * a finished row's result say, and how a whole run went. + */ +import type { + DocumentImportOutcome, + DocumentSourceResultDto, +} from "@/lib/documents/sources/types"; +import type { WrittenOutcome } from "@/lib/outcome/written-outcome"; + +export type RowRun = + | { state: "waiting" } + | { state: "working" } + | { state: DocumentImportOutcome; linked: boolean } + | { state: "failed"; message: string }; + +/** + * Whether a result row can be ticked: a new one always; one deleted here too, + * since ticking it asks for it back (`revive`); an already imported one only + * when the pick does something with it: link it, or hand it to the form. + */ +export function isPickable( + row: DocumentSourceResultDto, + canTakeImported: boolean, +): boolean { + if (row.state === "imported") return canTakeImported; + return true; +} + +/** The rows the vault holds as deleted: ticking one asks for it back. */ +export function deletedSourceIds(rows: DocumentSourceResultDto[]): Set { + return new Set( + rows.filter((r) => r.state === "deleted").map((r) => r.sourceId), + ); +} + +/** + * The badge on a row the vault already holds. A deleted one still in Recently + * deleted (the search names it) says so; one removed for good says deleted. + */ +export function heldBadgeLabel( + t: (key: string) => string, + row: DocumentSourceResultDto, +): string { + if (row.state === "imported") { + return t("documents.sourcePicker.stateImported"); + } + return row.documentId + ? t("documents.sourcePicker.stateInTrash") + : t("documents.sourcePicker.stateDeleted"); +} + +/** How the run went, in the shared outcome vocabulary. */ +export function runOutcome(counts: { + imported: number; + present: number; + failed: number; +}): WrittenOutcome { + const held = counts.imported + counts.present; + if (counts.failed === 0) return held > 0 ? "success" : "empty"; + return held > 0 ? "partial" : "failed"; +} + +/** A row's result in words, or null while it has none yet. */ +export function runLabel( + t: (key: string, params?: Record) => string, + run: RowRun, +): string | null { + switch (run.state) { + case "waiting": + case "working": + return null; + case "failed": + return run.message; + case "deleted": + return t("documents.sourcePicker.result.deleted"); + case "restored": + return run.linked + ? t("documents.sourcePicker.result.restoredLinked") + : t("documents.sourcePicker.result.restored"); + case "imported": + return run.linked + ? t("documents.sourcePicker.result.importedLinked") + : t("documents.sourcePicker.result.imported"); + case "duplicate": + return run.linked + ? t("documents.sourcePicker.result.duplicateLinked") + : t("documents.sourcePicker.result.duplicate"); + } +} diff --git a/src/lib/documents/source-key.ts b/src/lib/documents/source-key.ts index f089aa3b4..a2a5d8570 100644 --- a/src/lib/documents/source-key.ts +++ b/src/lib/documents/source-key.ts @@ -88,6 +88,26 @@ export async function findSourceKey( return purged ? { state: "deleted", id: null } : null; } +/** + * Forget the ledger's record that the person deleted the document a key named + * (`DocumentImportKey`, written by the purge), matching instances the way + * {@link findSourceKey} does. Only the person's own explicit "bring it back" + * from the document picker calls this, and only once the document is in the + * vault again; an importer's upload never does, so a deletion keeps holding + * against every automatic re-send. + */ +export async function forgetDeletedSourceKey( + userId: string, + sourceSystem: DocumentSourceSystemValue, + sourceId: string, + sourceInstance: string | null = null, +): Promise { + const { count } = await prisma.documentImportKey.deleteMany({ + where: { userId, sourceSystem, sourceId, ...instanceMatch(sourceInstance) }, + }); + return count; +} + /** * How many further keys one document may collect. Real archives hold a file * under one or two ids; the cap stops a leaked token from growing the table @@ -142,9 +162,14 @@ export function checkSourceLookupRateLimit( ); } -/** What the vault holds for one source key, as the picker lists it. */ +/** + * What the vault holds for one source key, as the picker lists it. A deleted + * key names its document while that is still in Recently deleted (restorable) + * and null once the purge removed it. + */ export type SourceKeyState = - { state: "imported"; documentId: string } | { state: "deleted" }; + | { state: "imported"; documentId: string } + | { state: "deleted"; documentId: string | null }; /** * {@link findSourceKey} for a page of results at once: the same three places, @@ -205,12 +230,14 @@ export async function findSourceKeyStates( // Lowest precedence first, so the document's own key wins, as it does in // `findSourceKey`. - for (const row of purged) states.set(row.sourceId, { state: "deleted" }); + for (const row of purged) { + states.set(row.sourceId, { state: "deleted", documentId: null }); + } for (const row of wildcardFirst(aliases)) { states.set( row.sourceId, row.document.deletedAt - ? { state: "deleted" } + ? { state: "deleted", documentId: row.document.id } : { state: "imported", documentId: row.document.id }, ); } @@ -219,7 +246,7 @@ export async function findSourceKeyStates( states.set( row.sourceId, row.deletedAt - ? { state: "deleted" } + ? { state: "deleted", documentId: row.id } : { state: "imported", documentId: row.id }, ); } diff --git a/src/lib/documents/sources/types.ts b/src/lib/documents/sources/types.ts index 3fdbfd1aa..cf5eac095 100644 --- a/src/lib/documents/sources/types.ts +++ b/src/lib/documents/sources/types.ts @@ -96,7 +96,11 @@ export interface DocumentSourceResultDto { tags: string[]; sizeBytes: number | null; state: DocumentPickerResultState; - /** The vault document when `state` is `imported`. */ + /** + * The vault document when `state` is `imported`. When `state` is `deleted`, + * the document while it is still in Recently deleted, and null once it has + * been removed for good. + */ documentId: string | null; } @@ -106,7 +110,12 @@ export interface DocumentSourceSearchDto { hasMore: boolean; } -export type DocumentImportOutcome = "imported" | "duplicate" | "deleted"; +/** + * `restored` answers only an import that asked to bring a deleted document + * back (`revive`): the one still in Recently deleted was restored. + */ +export type DocumentImportOutcome = + "imported" | "duplicate" | "deleted" | "restored"; export interface DocumentSourceImportDto { outcome: DocumentImportOutcome; diff --git a/src/lib/openapi/routes/document-sources.ts b/src/lib/openapi/routes/document-sources.ts index 0b97d283c..0b71ce532 100644 --- a/src/lib/openapi/routes/document-sources.ts +++ b/src/lib/openapi/routes/document-sources.ts @@ -67,7 +67,12 @@ const resultSchema = z tags: z.array(z.string()), sizeBytes: z.number().int().nullable(), state: z.enum(["new", "imported", "deleted"]), - documentId: z.string().nullable(), + documentId: z + .string() + .nullable() + .describe( + "The vault document for `imported`. For `deleted`, the document while it is still in Recently deleted (restorable), null once it has been removed for good.", + ), }) .meta({ id: "DocumentSourceResult" }); @@ -81,7 +86,11 @@ const searchSchema = z const importResultSchema = z .object({ - outcome: z.enum(["imported", "duplicate", "deleted"]), + outcome: z + .enum(["imported", "duplicate", "deleted", "restored"]) + .describe( + "`restored` only answers a request that sent `revive: true`: the deleted document was still in Recently deleted and is back in the vault.", + ), documentId: z.string().nullable(), linked: z.boolean(), }) @@ -335,7 +344,7 @@ export const documentSourcePaths: NonNullable = { tags: ["Documents"], summary: "Import one picked document", description: - "`sourceId` must be an id the system issues (Paperless-ngx: digits; Papra: letters, digits, `_`, `-`), else 422 before anything is fetched. Fetches one document from the source and stores it in the vault through the same path as an upload, with `sourceSystem` / `sourceId` / `sourceInstance` (the connection's origin) provenance; the same id from another instance is another document. One import per key runs at a time; a concurrent second one waits and then answers `duplicate`. The key check counts toward the lookup allowance, the fetch toward the search allowance, and only a file that arrived toward the upload allowance. A source key the vault already holds is answered without a download: `duplicate` for a live document (same id), `deleted` for one the person deleted in HealthLog, which is never stored again. Bytes already in the vault under another key are `duplicate` too, and the key is remembered for that document. `link` files the document against the record the picker was opened from (a condition, a visit or a vaccination, which must be the caller's own and live; checked before anything is fetched), including a duplicate. `kind` sets the document type (default OTHER). Automatic AI reading follows the person's own setting. Counts against the person's upload allowance (60 an hour) and storage quota. 201 for a new document, 200 otherwise. Cookie session only; not delegable.", + "`sourceId` must be an id the system issues (Paperless-ngx: digits; Papra: letters, digits, `_`, `-`), else 422 before anything is fetched. Fetches one document from the source and stores it in the vault through the same path as an upload, with `sourceSystem` / `sourceId` / `sourceInstance` (the connection's origin) provenance; the same id from another instance is another document. One import per key runs at a time; a concurrent second one waits and then answers `duplicate`. The key check counts toward the lookup allowance, the fetch toward the search allowance, and only a file that arrived toward the upload allowance. A source key the vault already holds is answered without a download: `duplicate` for a live document (same id), `deleted` for one the person deleted in HealthLog, which is not stored again unless the request sends `revive: true`: then a document still in Recently deleted is restored (`restored`), and one already removed for good is fetched and stored again, its key leaving the import ledger. A token upload (`POST /api/documents/inbound`) has no such switch, so an importer never brings a deleted document back. Bytes already in the vault under another key are `duplicate` too, and the key is remembered for that document. `link` files the document against the record the picker was opened from (a condition, a visit or a vaccination, which must be the caller's own and live; checked before anything is fetched), including a duplicate. `kind` sets the document type (default OTHER). Automatic AI reading follows the person's own setting. Counts against the person's upload allowance (60 an hour) and storage quota. 201 for a new document, 200 otherwise. Cookie session only; not delegable.", security: cookieOnly, parameters: [systemParameter], requestBody: { @@ -358,7 +367,7 @@ export const documentSourcePaths: NonNullable = { }, "200": { description: - "Already held (`duplicate`) or deleted in HealthLog (`deleted`).", + "Already held (`duplicate`), deleted in HealthLog (`deleted`), or brought back from Recently deleted (`restored`).", content: { "application/json": { schema: dataEnvelope( diff --git a/src/lib/validations/document-sources.ts b/src/lib/validations/document-sources.ts index 4034d4c39..24bbaae33 100644 --- a/src/lib/validations/document-sources.ts +++ b/src/lib/validations/document-sources.ts @@ -84,6 +84,14 @@ export const documentSourceImportSchema = z.object({ id: z.string().trim().min(1).max(64), }) .optional(), + /** + * The person chose to bring back a document they deleted in HealthLog. One + * still in Recently deleted is restored (`restored`); one already removed + * for good is fetched and stored again, and the import ledger forgets its + * key. Without it a deleted document is answered `deleted` and left alone, + * as an importer's upload always is. + */ + revive: z.boolean().optional(), }); export type DocumentSourceImportInput = z.infer< typeof documentSourceImportSchema diff --git a/tests/integration/document-sources.test.ts b/tests/integration/document-sources.test.ts index 70776ef58..7793fab8d 100644 --- a/tests/integration/document-sources.test.ts +++ b/tests/integration/document-sources.test.ts @@ -522,9 +522,10 @@ describe("search and import", () => { where: { id: receipt.documentId }, data: { deletedAt: new Date() }, }); + // Deleted but still in Recently deleted: the search names the document. expect( (await (await search("q=blood")).json()).data.results[0], - ).toMatchObject({ state: "deleted", documentId: null }); + ).toMatchObject({ state: "deleted", documentId: receipt.documentId }); const revived = await importDoc({ sourceId: "41" }); expect(revived.status).toBe(200); expect((await revived.json()).data.outcome).toBe("deleted"); @@ -532,6 +533,90 @@ describe("search and import", () => { expect(await prisma.inboundDocument.count()).toBe(1); }); + it("brings a deleted document back only when asked: restored from Recently deleted, or imported again after the purge", async () => { + await seedOwner(); + await connect(); + const prisma = getPrismaClient(); + + const first = await importDoc({ sourceId: "41" }); + const id = (await first.json()).data.documentId as string; + await prisma.inboundDocument.update({ + where: { id }, + data: { deletedAt: new Date(), status: "DISCARDED" }, + }); + + // Still in Recently deleted: `revive` restores the same document, and + // nothing is downloaded. + const restored = await importDoc({ sourceId: "41", revive: true }); + expect(restored.status).toBe(200); + expect((await restored.json()).data).toEqual({ + outcome: "restored", + documentId: id, + linked: false, + }); + expect(downloads()).toBe(1); + expect( + (await prisma.inboundDocument.findUniqueOrThrow({ where: { id } })) + .deletedAt, + ).toBeNull(); + expect( + await prisma.auditLog.count({ + where: { userId, action: "documents.inbound.restore" }, + }), + ).toBe(1); + + // Deleted again and purged: the key is in the ledger, and an import that + // does not ask is still answered "deleted". + await prisma.inboundDocument.update({ + where: { id }, + data: { deletedAt: new Date(Date.now() - 31 * 86_400_000) }, + }); + const { purgeExpiredDocumentTombstones } = + await import("@/lib/jobs/document-purge"); + expect(await purgeExpiredDocumentTombstones(prisma as never)).toBe(1); + expect( + (await (await search("q=blood")).json()).data.results[0], + ).toMatchObject({ state: "deleted", documentId: null }); + const left = await importDoc({ sourceId: "41" }); + expect((await left.json()).data).toEqual({ + outcome: "deleted", + documentId: null, + linked: false, + }); + expect(downloads()).toBe(1); + + // Asked: fetched and stored again, and the ledger forgets the key. + const again = await importDoc({ sourceId: "41", revive: true }); + expect(again.status).toBe(201); + const back = (await again.json()).data; + expect(back).toMatchObject({ outcome: "imported", linked: false }); + expect(back.documentId).not.toBe(id); + expect(downloads()).toBe(2); + expect(await prisma.documentImportKey.count({ where: { userId } })).toBe(0); + expect( + (await (await search("q=blood")).json()).data.results[0], + ).toMatchObject({ state: "imported", documentId: back.documentId }); + }); + + it("keeps the deletion when a revived import cannot fetch the file", async () => { + await seedOwner(); + await connect(); + const prisma = getPrismaClient(); + await prisma.documentImportKey.create({ + data: { + userId, + sourceSystem: "PAPERLESS", + sourceId: "42", + sourceInstance: ORIGIN, + }, + }); + fake.downloadStatus = 500; + const failed = await importDoc({ sourceId: "42", revive: true }); + expect(failed.status).toBeGreaterThanOrEqual(400); + expect(await prisma.documentImportKey.count({ where: { userId } })).toBe(1); + expect(await prisma.inboundDocument.count()).toBe(0); + }); + it("answers a file already uploaded by hand as a duplicate and remembers the key", async () => { await seedOwner(); await connect();