diff --git a/docs/api/openapi.yaml b/docs/api/openapi.yaml index cf89cde46..b92152575 100644 --- a/docs/api/openapi.yaml +++ b/docs/api/openapi.yaml @@ -25487,9 +25487,12 @@ paths: waits and then answers `duplicate`. The key check counts toward the lookup allowance, the fetch toward the search allowance, and only a file that arrived toward the upload allowance. A source key the vault already holds is answered without a download: `duplicate` for a live document (same id), `deleted` for one the person deleted - in HealthLog, which is never stored again. Bytes already in the vault under another key are `duplicate` too, and - the key is remembered for that document. `link` files the document against the record the picker was opened from - (a condition, a visit or a vaccination, which must be the caller's own and live; checked before anything is + in HealthLog, which is not stored again unless the request sends `revive: true`: then a document still in + Recently deleted is restored (`restored`), and one already removed for good is fetched and stored again, its key + leaving the import ledger. A token upload (`POST /api/documents/inbound`) has no such switch, so an importer + never brings a deleted document back. Bytes already in the vault under another key are `duplicate` too, and the + key is remembered for that document. `link` files the document against the record the picker was opened from (a + condition, a visit or a vaccination, which must be the caller's own and live; checked before anything is fetched), including a duplicate. `kind` sets the document type (default OTHER). Automatic AI reading follows the person's own setting. Counts against the person's upload allowance (60 an hour) and storage quota. 201 for a new document, 200 otherwise. Cookie session only; not delegable." @@ -25537,11 +25540,14 @@ paths: required: - kind - id + revive: + type: boolean required: - sourceId responses: "200": - description: Already held (`duplicate`) or deleted in HealthLog (`deleted`). + description: Already held (`duplicate`), deleted in HealthLog (`deleted`), or brought back from Recently deleted + (`restored`). content: application/json: schema: @@ -90451,6 +90457,8 @@ components: - imported - deleted documentId: + description: The vault document for `imported`. For `deleted`, the document while it is still in Recently deleted + (restorable), null once it has been removed for good. type: - string - "null" @@ -90489,6 +90497,9 @@ components: - imported - duplicate - deleted + - restored + description: "`restored` only answers a request that sent `revive: true`: the deleted document was still in Recently + deleted and is back in the vault." documentId: type: - string diff --git a/docs/self-hosting/document-sources.md b/docs/self-hosting/document-sources.md index 4a30cdd2c..e0be3e5cc 100644 --- a/docs/self-hosting/document-sources.md +++ b/docs/self-hosting/document-sources.md @@ -82,13 +82,18 @@ limit for a document). - Imports count against the person's normal upload allowance (60 an hour) and storage quota. -- A document that is already in HealthLog is not stored twice, and one the - person deleted in HealthLog is not brought back by picking it again. Each - import is keyed by system, instance (the archive's origin) and document id, - so the same id on two Paperless-ngx instances is two documents; a key - imported before v1.39.3 carries no instance and matches any instance. These - are the same rules the document token and `scripts/import-documents.mjs` - follow. +- A document that is already in HealthLog is not stored twice. Each import is + keyed by system, instance (the archive's origin) and document id, so the + same id on two Paperless-ngx instances is two documents; a key imported + before v1.39.3 carries no instance and matches any instance. These are the + same rules the document token and `scripts/import-documents.mjs` follow. +- A document the person deleted in HealthLog is never brought back by the + document token or the import script, however often they re-send it. The + person can bring it back themselves: from Recently deleted on the Documents + page for 30 days, or by picking it again in the picker, which says "In Recently + deleted" or "Deleted in HealthLog" on such a row. Picking one still in + Recently deleted restores that same document; picking one already removed + for good imports it again. - A picked document follows the person's own automatic AI reading setting. - Only the owner of a record can connect an archive or import from it. People with shared access to a record, whatever their level, do not see the picker. diff --git a/messages/de.json b/messages/de.json index 4ef2f6ce8..bd07d9f7a 100644 --- a/messages/de.json +++ b/messages/de.json @@ -11890,6 +11890,7 @@ "loadMore": "Mehr laden", "stateImported": "In HealthLog", "stateDeleted": "In HealthLog gelöscht", + "stateInTrash": "Unter Zuletzt gelöscht", "selectedCount": "{count} von höchstens {max} ausgewählt", "importAction": "{count} importieren", "importing": "Import läuft, ein Dokument nach dem anderen …", @@ -11903,7 +11904,9 @@ "importedLinked": "Importiert und verknüpft", "duplicate": "Schon in HealthLog", "duplicateLinked": "Schon in HealthLog, jetzt verknüpft", - "deleted": "In HealthLog gelöscht, bleibt gelöscht" + "deleted": "In HealthLog gelöscht, bleibt gelöscht", + "restored": "Aus Zuletzt gelöscht wiederhergestellt", + "restoredLinked": "Aus Zuletzt gelöscht wiederhergestellt und verknüpft" }, "errors": { "unavailable": "Dokumentarchive sind auf diesem Server nicht freigeschaltet.", diff --git a/messages/en.json b/messages/en.json index 71c85501f..da7365a1e 100644 --- a/messages/en.json +++ b/messages/en.json @@ -11890,6 +11890,7 @@ "loadMore": "Load more", "stateImported": "In HealthLog", "stateDeleted": "Deleted in HealthLog", + "stateInTrash": "In Recently deleted", "selectedCount": "{count} of up to {max} selected", "importAction": "Import {count}", "importing": "Importing, one document at a time…", @@ -11903,7 +11904,9 @@ "importedLinked": "Imported and linked", "duplicate": "Already in HealthLog", "duplicateLinked": "Already in HealthLog, now linked", - "deleted": "Deleted in HealthLog, left alone" + "deleted": "Deleted in HealthLog, left alone", + "restored": "Restored from Recently deleted", + "restoredLinked": "Restored from Recently deleted and linked" }, "errors": { "unavailable": "Document archives are not enabled on this server.", diff --git a/messages/es.json b/messages/es.json index a90623fc8..a87864f1c 100644 --- a/messages/es.json +++ b/messages/es.json @@ -11890,6 +11890,7 @@ "loadMore": "Cargar más", "stateImported": "En HealthLog", "stateDeleted": "Eliminado en HealthLog", + "stateInTrash": "En Eliminados recientemente", "selectedCount": "{count} de un máximo de {max} seleccionados", "importAction": "Importar {count}", "importing": "Importando, un documento tras otro…", @@ -11903,7 +11904,9 @@ "importedLinked": "Importado y vinculado", "duplicate": "Ya está en HealthLog", "duplicateLinked": "Ya está en HealthLog, ahora vinculado", - "deleted": "Eliminado en HealthLog, se deja así" + "deleted": "Eliminado en HealthLog, se deja así", + "restored": "Restaurado desde Eliminados recientemente", + "restoredLinked": "Restaurado desde Eliminados recientemente y vinculado" }, "errors": { "unavailable": "Los archivos de documentos no están habilitados en este servidor.", diff --git a/messages/fr.json b/messages/fr.json index 7d93dc773..61afc2d54 100644 --- a/messages/fr.json +++ b/messages/fr.json @@ -11890,6 +11890,7 @@ "loadMore": "Charger plus", "stateImported": "Dans HealthLog", "stateDeleted": "Supprimé dans HealthLog", + "stateInTrash": "Dans Supprimés récemment", "selectedCount": "{count} sélectionnés sur {max} au plus", "importAction": "Importer {count}", "importing": "Import en cours, un document après l'autre…", @@ -11903,7 +11904,9 @@ "importedLinked": "Importé et lié", "duplicate": "Déjà dans HealthLog", "duplicateLinked": "Déjà dans HealthLog, désormais lié", - "deleted": "Supprimé dans HealthLog, laissé tel quel" + "deleted": "Supprimé dans HealthLog, laissé tel quel", + "restored": "Restauré depuis Supprimés récemment", + "restoredLinked": "Restauré depuis Supprimés récemment et lié" }, "errors": { "unavailable": "Les archives de documents ne sont pas activées sur ce serveur.", diff --git a/messages/it.json b/messages/it.json index 1e6f6cce6..c1affc89d 100644 --- a/messages/it.json +++ b/messages/it.json @@ -11890,6 +11890,7 @@ "loadMore": "Carica altri", "stateImported": "Già in HealthLog", "stateDeleted": "Eliminato in HealthLog", + "stateInTrash": "In Eliminati di recente", "selectedCount": "{count} selezionati su un massimo di {max}", "importAction": "Importa {count}", "importing": "Importazione in corso, un documento alla volta…", @@ -11903,7 +11904,9 @@ "importedLinked": "Importato e collegato", "duplicate": "Già in HealthLog", "duplicateLinked": "Già in HealthLog, ora collegato", - "deleted": "Eliminato in HealthLog, lasciato così" + "deleted": "Eliminato in HealthLog, lasciato così", + "restored": "Ripristinato da Eliminati di recente", + "restoredLinked": "Ripristinato da Eliminati di recente e collegato" }, "errors": { "unavailable": "Gli archivi di documenti non sono abilitati su questo server.", diff --git a/messages/ko.json b/messages/ko.json index dc0f9212b..7c9a30697 100644 --- a/messages/ko.json +++ b/messages/ko.json @@ -11890,6 +11890,7 @@ "loadMore": "더 불러오기", "stateImported": "HealthLog에 있음", "stateDeleted": "HealthLog에서 삭제됨", + "stateInTrash": "최근 삭제됨에 있음", "selectedCount": "최대 {max}개 중 {count}개 선택", "importAction": "{count}개 가져오기", "importing": "문서를 하나씩 가져오는 중…", @@ -11903,7 +11904,9 @@ "importedLinked": "가져오고 연결함", "duplicate": "이미 HealthLog에 있음", "duplicateLinked": "이미 HealthLog에 있음, 이제 연결됨", - "deleted": "HealthLog에서 삭제됨, 그대로 둠" + "deleted": "HealthLog에서 삭제됨, 그대로 둠", + "restored": "최근 삭제됨에서 복구했어요", + "restoredLinked": "최근 삭제됨에서 복구하고 연결했어요" }, "errors": { "unavailable": "이 서버에서는 문서 보관함이 활성화되어 있지 않습니다.", diff --git a/messages/nl.json b/messages/nl.json index a831c0207..4441b4dd8 100644 --- a/messages/nl.json +++ b/messages/nl.json @@ -11890,6 +11890,7 @@ "loadMore": "Meer laden", "stateImported": "Staat in HealthLog", "stateDeleted": "Verwijderd in HealthLog", + "stateInTrash": "In Recent verwijderd", "selectedCount": "{count} van maximaal {max} geselecteerd", "importAction": "{count} importeren", "importing": "Importeren, één document tegelijk…", @@ -11903,7 +11904,9 @@ "importedLinked": "Geïmporteerd en gekoppeld", "duplicate": "Staat al in HealthLog", "duplicateLinked": "Staat al in HealthLog, nu gekoppeld", - "deleted": "Verwijderd in HealthLog, ongemoeid gelaten" + "deleted": "Verwijderd in HealthLog, ongemoeid gelaten", + "restored": "Hersteld uit Recent verwijderd", + "restoredLinked": "Hersteld uit Recent verwijderd en gekoppeld" }, "errors": { "unavailable": "Documentarchieven zijn niet ingeschakeld op deze server.", diff --git a/messages/pl.json b/messages/pl.json index 679c7ee7d..8c84eafc1 100644 --- a/messages/pl.json +++ b/messages/pl.json @@ -11890,6 +11890,7 @@ "loadMore": "Wczytaj więcej", "stateImported": "W HealthLog", "stateDeleted": "Usunięty w HealthLog", + "stateInTrash": "W sekcji Ostatnio usunięte", "selectedCount": "Wybrano {count} z maksymalnie {max}", "importAction": "Importuj ({count})", "importing": "Importowanie, dokument po dokumencie…", @@ -11903,7 +11904,9 @@ "importedLinked": "Zaimportowano i powiązano", "duplicate": "Już jest w HealthLog", "duplicateLinked": "Już jest w HealthLog, teraz powiązany", - "deleted": "Usunięty w HealthLog, pozostaje usunięty" + "deleted": "Usunięty w HealthLog, pozostaje usunięty", + "restored": "Przywrócono z sekcji Ostatnio usunięte", + "restoredLinked": "Przywrócono z sekcji Ostatnio usunięte i powiązano" }, "errors": { "unavailable": "Archiwa dokumentów nie są włączone na tym serwerze.", diff --git a/messages/pt-BR.json b/messages/pt-BR.json index d19fdec09..d325f0552 100644 --- a/messages/pt-BR.json +++ b/messages/pt-BR.json @@ -11890,6 +11890,7 @@ "loadMore": "Carregar mais", "stateImported": "No HealthLog", "stateDeleted": "Excluído no HealthLog", + "stateInTrash": "Em Excluídos recentemente", "selectedCount": "{count} de até {max} selecionados", "importAction": "Importar {count}", "importing": "Importando, um documento por vez…", @@ -11903,7 +11904,9 @@ "importedLinked": "Importado e vinculado", "duplicate": "Já no HealthLog", "duplicateLinked": "Já no HealthLog, agora vinculado", - "deleted": "Excluído no HealthLog, deixado como está" + "deleted": "Excluído no HealthLog, deixado como está", + "restored": "Restaurado de Excluídos recentemente", + "restoredLinked": "Restaurado de Excluídos recentemente e vinculado" }, "errors": { "unavailable": "Arquivos de documentos não estão habilitados neste servidor.", diff --git a/src/__tests__/baselines/source-function-lines.json b/src/__tests__/baselines/source-function-lines.json index 7b0728781..3a8b2011a 100644 --- a/src/__tests__/baselines/source-function-lines.json +++ b/src/__tests__/baselines/source-function-lines.json @@ -1,7 +1,7 @@ { "$comment": "Production functions over 200 lines, keyed \"file :: name\", and their length. Maintained by src/__tests__/source-size-ratchet.test.ts.", "count": 351, - "total": 113742, + "total": 113740, "entries": { "app/achievements/page.tsx :: AchievementsPage": 206, "app/api/admin/backups/upload/route.ts :: apiHandler(arg)": 314, @@ -100,7 +100,7 @@ "components/documents/document-record-links.tsx :: DocumentRecordLinks": 202, "components/documents/document-timeline.tsx :: DocumentTimeline": 314, "components/documents/documents-view.tsx :: DocumentsView": 303, - "components/documents/sources/document-source-picker.tsx :: DocumentSourcePicker": 565, + "components/documents/sources/document-source-picker.tsx :: DocumentSourcePicker": 563, "components/illness/log-day-sheet.tsx :: LogDaySheet": 265, "components/illness/new-episode-sheet.tsx :: NewEpisodeSheet": 264, "components/insights/coach-panel/attachment-picker.tsx :: AttachmentPicker": 208, diff --git a/src/app/api/documents/sources/[system]/import/route.ts b/src/app/api/documents/sources/[system]/import/route.ts index a5f4cb90f..854180ac0 100644 --- a/src/app/api/documents/sources/[system]/import/route.ts +++ b/src/app/api/documents/sources/[system]/import/route.ts @@ -11,7 +11,13 @@ * 2. the source key answers first, without a download: a document already * imported (or already holding this key as an alias) is a `duplicate`, * and one the person deleted here is `deleted` — nothing is fetched and - * nothing is stored, exactly as the token upload answers the same key; + * nothing is stored, exactly as the token upload answers the same key. + * Unless the person asked to bring it back (`revive`, sent only for a + * row they picked knowing it was deleted): a document still in Recently + * deleted is restored (`restored`, nothing fetched), and one the purge + * already removed is fetched and stored like a new one, after which the + * import ledger forgets its key. The token upload has no such switch, so + * a deletion keeps holding against every automatic re-send; * 3. only then is the source asked (on the search allowance, like every * request to it) and an upload slot taken. One import per key and person * runs at a time under an advisory lock, so a double click downloads the @@ -37,6 +43,7 @@ import { NextRequest, NextResponse } from "next/server"; import { apiHandler, requireAuth } from "@/lib/api-handler"; import { apiError, getClientIp } from "@/lib/api-response"; +import { auditLog } from "@/lib/auth/audit"; import { parseJsonBody } from "@/lib/api/body"; import { getAiCapability } from "@/lib/ai/capabilities/gate"; import { prisma } from "@/lib/db"; @@ -49,6 +56,7 @@ import { import { checkSourceLookupRateLimit, findSourceKey, + forgetDeletedSourceKey, type SourceKeyMatch, } from "@/lib/documents/source-key"; import type { SourceClient } from "@/lib/documents/sources/clients"; @@ -71,6 +79,7 @@ import { acquireDocumentUploadSlot, resolveDocumentLimits, } from "@/lib/documents/upload-policy"; +import { restoreDocumentTombstone } from "@/lib/documents/tombstones"; import { linkTargets } from "@/lib/links"; import { annotate } from "@/lib/logging/context"; import { checkRateLimit, rateLimitHeaders } from "@/lib/rate-limit"; @@ -148,18 +157,82 @@ function respond( ); } -/** A key the vault already holds: link a live one, never revive a deleted one. */ +/** The key the request names, for the ledger. */ +interface PickedKey { + system: DocumentPickerSystem; + sourceId: string; + instance: string; +} + +/** + * A key the vault already holds: link a live one; leave a deleted one alone + * unless the person asked to bring it back (`revive`). Then a document still + * in Recently deleted is restored here, and for one the purge already removed + * this answers null: the caller fetches and stores it again. + */ async function answerHeldKey( - system: string, + request: Request, + key: PickedKey, userId: string, link: LinkTarget | undefined, match: SourceKeyMatch, -): Promise { + revive: boolean, +): Promise { if (match.state === "deleted") { - return respond(system, "deleted", match.id, false); + if (!revive) return respond(key.system, "deleted", match.id, false); + if (match.id === null) return null; + return restoreFromPicker(request, key, userId, link, match.id); } const linked = await linkDocument(userId, link, match.document.id); - return respond(system, "duplicate", match.document.id, linked); + return respond(key.system, "duplicate", match.document.id, linked); +} + +/** + * Bring a picked document back out of Recently deleted, through the same + * restore the vault uses (which also clears the ledger for the document's own + * keys), and forget the picked key's ledger entry the way a re-import would. + */ +async function restoreFromPicker( + request: Request, + key: PickedKey, + userId: string, + link: LinkTarget | undefined, + documentId: string, +): Promise { + const outcome = await restoreDocumentTombstone(userId, documentId); + switch (outcome.kind) { + case "gone": + // Purged between the key check and the restore: nothing to bring back + // from Recently deleted, and nothing is fetched on this request. + return respond(key.system, "deleted", null, false); + case "duplicate": { + // The same bytes are live under another document: that one is the + // person's copy, so the pick links it, like any duplicate. + const existing = outcome.existingId; + const linked = existing + ? await linkDocument(userId, link, existing) + : false; + return respond(key.system, "duplicate", existing, linked); + } + case "live": + case "restored": { + await forgetDeletedSourceKey( + userId, + key.system, + key.sourceId, + key.instance, + ); + if (outcome.kind === "restored") { + await auditLog("documents.inbound.restore", { + userId, + ipAddress: getClientIp(request), + details: { documentId, via: "documentPicker" }, + }); + } + const linked = await linkDocument(userId, link, documentId); + return respond(key.system, "restored", documentId, linked); + } + } } export const POST = apiHandler( @@ -177,6 +250,7 @@ export const POST = apiHandler( }); if (!parsed.ok) return parsed.error; const { sourceId, kind, link } = parsed.data; + const revive = parsed.data.revive === true; // Only an id this system could have issued goes into a request path. if (!isSourceDocumentId(system, sourceId)) { return apiError("Not a document id of this archive.", 422); @@ -210,8 +284,19 @@ export const POST = apiHandler( rateLimitHeaders(lookupRl), ); } + const key: PickedKey = { system, sourceId, instance }; const held = await findSourceKey(userId, system, sourceId, instance); - if (held) return answerHeldKey(system, userId, link, held); + if (held) { + const answered = await answerHeldKey( + request, + key, + userId, + link, + held, + revive, + ); + if (answered) return answered; + } // A request to the source counts against the search allowance like any // other; the upload allowance is charged only once a file arrived. @@ -249,7 +334,17 @@ export const POST = apiHandler( FROM pg_advisory_xact_lock(hashtextextended(${`documents-import:${userId}:${system}:${instance}:${sourceId}`}, 0)) `; const raced = await findSourceKey(userId, system, sourceId, instance); - if (raced) return answerHeldKey(system, userId, link, raced); + if (raced) { + const answered = await answerHeldKey( + request, + key, + userId, + link, + raced, + revive, + ); + if (answered) return answered; + } return importOne({ request, client, @@ -259,6 +354,7 @@ export const POST = apiHandler( sourceId, kind: kind ?? null, link, + revive, }); }, { maxWait: IMPORT_LOCK_WAIT_MS, timeout: IMPORT_LOCK_HOLD_MS }, @@ -279,9 +375,21 @@ async function importOne(args: { sourceId: string; kind: InboundDocumentKindValue | null; link: LinkTarget | undefined; + /** The person asked to bring back a document they deleted here. */ + revive: boolean; }): Promise { - const { request, client, userId, system, instance, sourceId, kind, link } = - args; + const { + request, + client, + userId, + system, + instance, + sourceId, + kind, + link, + revive, + } = args; + const key: PickedKey = { system, sourceId, instance }; const limits = await resolveDocumentLimits(userId); let meta; let file; @@ -326,16 +434,36 @@ async function importOne(args: { documentAi: () => getAiCapability("documentAi"), }); + // A key the purge removed was kept in the ledger; once the person brought + // the document back by hand, the ledger stops saying it is deleted. Done + // only after the document is in the vault again, so a fetch that failed + // leaves the deletion standing. + const forgetIfRevived = async () => { + if (revive) + await forgetDeletedSourceKey(userId, system, sourceId, instance); + }; + switch (result.kind) { case "sourceKey": - return answerHeldKey(system, userId, link, result.match); + return ( + (await answerHeldKey( + request, + key, + userId, + link, + result.match, + revive, + )) ?? respond(system, "deleted", null, false) + ); case "duplicate": { // The same bytes were already in the vault (an upload, or another // archive); the key is now remembered for that document. + await forgetIfRevived(); const linked = await linkDocument(userId, link, result.document.id); return respond(system, "duplicate", result.document.id, linked); } case "stored": { + await forgetIfRevived(); const linked = await linkDocument(userId, link, result.document.id); return respond(system, "imported", result.document.id, linked); } diff --git a/src/app/api/documents/sources/[system]/search/route.ts b/src/app/api/documents/sources/[system]/search/route.ts index edaf7a9c3..22e4138b4 100644 --- a/src/app/api/documents/sources/[system]/search/route.ts +++ b/src/app/api/documents/sources/[system]/search/route.ts @@ -132,7 +132,9 @@ export const GET = apiHandler( tags: item.tags, sizeBytes: item.sizeBytes, state: held?.state ?? "new", - documentId: held?.state === "imported" ? held.documentId : null, + // The vault document for `imported`; for `deleted`, the one still in + // Recently deleted (null once the purge removed it). + documentId: held?.documentId ?? null, }; }); diff --git a/src/components/documents/sources/__tests__/picker-rows.test.ts b/src/components/documents/sources/__tests__/picker-rows.test.ts new file mode 100644 index 000000000..ac3d310b2 --- /dev/null +++ b/src/components/documents/sources/__tests__/picker-rows.test.ts @@ -0,0 +1,85 @@ +import { describe, expect, it } from "vitest"; + +import type { DocumentSourceResultDto } from "@/lib/documents/sources/types"; + +import { + deletedSourceIds, + heldBadgeLabel, + isPickable, + runLabel, + runOutcome, +} from "../picker-rows"; + +function row( + over: Partial = {}, +): DocumentSourceResultDto { + return { + sourceId: "41", + title: "Blood test", + date: null, + tags: [], + sizeBytes: null, + state: "new", + documentId: null, + ...over, + }; +} + +const t = (key: string) => key; + +describe("isPickable", () => { + it("ticks a new row and a deleted one (asking for it back)", () => { + expect(isPickable(row(), false)).toBe(true); + expect(isPickable(row({ state: "deleted", documentId: "d1" }), false)).toBe( + true, + ); + expect(isPickable(row({ state: "deleted" }), false)).toBe(true); + }); + + it("ticks an imported row only when the pick links it or hands it on", () => { + const imported = row({ state: "imported", documentId: "d1" }); + expect(isPickable(imported, false)).toBe(false); + expect(isPickable(imported, true)).toBe(true); + }); +}); + +describe("deletedSourceIds", () => { + it("names exactly the rows held as deleted, which the import revives", () => { + const ids = deletedSourceIds([ + row({ sourceId: "1" }), + row({ sourceId: "2", state: "deleted", documentId: "d2" }), + row({ sourceId: "3", state: "imported", documentId: "d3" }), + row({ sourceId: "4", state: "deleted" }), + ]); + expect([...ids].sort()).toEqual(["2", "4"]); + }); +}); + +describe("heldBadgeLabel", () => { + it("tells a document still in Recently deleted from one removed for good", () => { + expect(heldBadgeLabel(t, row({ state: "deleted", documentId: "d1" }))).toBe( + "documents.sourcePicker.stateInTrash", + ); + expect(heldBadgeLabel(t, row({ state: "deleted" }))).toBe( + "documents.sourcePicker.stateDeleted", + ); + expect(heldBadgeLabel(t, row({ state: "imported", documentId: "d" }))).toBe( + "documents.sourcePicker.stateImported", + ); + }); +}); + +describe("a restored row", () => { + it("reads as restored, linked or not", () => { + expect(runLabel(t, { state: "restored", linked: false })).toBe( + "documents.sourcePicker.result.restored", + ); + expect(runLabel(t, { state: "restored", linked: true })).toBe( + "documents.sourcePicker.result.restoredLinked", + ); + }); + + it("counts toward a successful run", () => { + expect(runOutcome({ imported: 1, present: 0, failed: 0 })).toBe("success"); + }); +}); diff --git a/src/components/documents/sources/document-source-picker.tsx b/src/components/documents/sources/document-source-picker.tsx index c492f5607..7b9732d7e 100644 --- a/src/components/documents/sources/document-source-picker.tsx +++ b/src/components/documents/sources/document-source-picker.tsx @@ -9,7 +9,11 @@ * field opens focused and debounces into the source's own name search. Tag and * date are filters beside it, not the entry point. Each result says what * HealthLog already holds under that source key: already imported, deleted - * here (not offered again: a deletion was a decision), or new. + * here, or new. A deleted one can be ticked too, and that is the person + * deciding again: one still in Recently deleted is restored (nothing is + * fetched), one already removed for good is imported afresh. The import sends + * `revive` only for those rows; an importer's upload never can, so a deletion + * still holds against every automatic re-send. * * The import runs one document per request, in order, so every row can show * its own progress and result, and one failure does not hide the others. A @@ -59,7 +63,6 @@ import { apiGet, apiPost } from "@/lib/api/api-fetch"; import { DOCUMENT_PICKER_MAX_SELECTION, slugForSystem, - type DocumentImportOutcome, type DocumentPickerLinkKind, type DocumentPickerSystem, type DocumentSourceImportDto, @@ -73,7 +76,6 @@ import { cn } from "@/lib/utils"; import type { InboundDocumentKindValue } from "@/lib/validations/inbound-documents"; import { WrittenOutcomeLine } from "@/components/outcome/written-outcome-line"; -import type { WrittenOutcome } from "@/lib/outcome/written-outcome"; import { formatBytes } from "@/lib/format/bytes"; import { @@ -84,6 +86,14 @@ import { import { Segmented } from "@/components/ui/segmented"; import { systemName } from "./use-document-sources"; import { InlineAction } from "@/components/ui/inline-action"; +import { + deletedSourceIds, + heldBadgeLabel, + isPickable, + runLabel, + runOutcome, + type RowRun, +} from "./picker-rows"; export interface DocumentPickerLink { kind: DocumentPickerLinkKind; @@ -103,12 +113,6 @@ export interface DocumentSourcePickerProps { onImported?: (documentIds: string[]) => void; } -type RowRun = - | { state: "waiting" } - | { state: "working" } - | { state: DocumentImportOutcome; linked: boolean } - | { state: "failed"; message: string }; - const SEARCH_DEBOUNCE_MS = 300; const IMPORT_TIMEOUT_MS = 120_000; @@ -242,11 +246,9 @@ export function DocumentSourcePicker({ return out; }, [search.data]); - // An already imported document is worth picking only when the pick does - // something with it: link it, or hand it to the form. const canTakeImported = Boolean(link || onImported); const selectable = (row: DocumentSourceResultDto) => - row.state === "new" || (row.state === "imported" && canTakeImported); + isPickable(row, canTakeImported); const toggle = (row: DocumentSourceResultDto) => { if (running || finished || !selectable(row)) return; @@ -269,6 +271,7 @@ export function DocumentSourcePicker({ setRunning(true); setStopMessage(null); const order = [...selected.keys()]; + const revive = deletedSourceIds(results); setRuns(new Map(order.map((id) => [id, { state: "waiting" }]))); const handed: string[] = []; const mark = (id: string, run: RowRun) => { @@ -287,6 +290,7 @@ export function DocumentSourcePicker({ sourceId, ...(kind ? { kind } : {}), ...(link ? { link } : {}), + ...(revive.has(sourceId) ? { revive: true } : {}), }, // The server may take up to a minute to fetch a large file from // the source; the default 15 s client timeout would abandon it. @@ -324,7 +328,7 @@ export function DocumentSourcePicker({ let present = 0; let failed = 0; for (const run of runs.values()) { - if (run.state === "imported") imported += 1; + if (run.state === "imported" || run.state === "restored") imported += 1; else if (run.state === "duplicate" || run.state === "deleted") present += 1; else if (run.state === "failed") failed += 1; @@ -643,9 +647,7 @@ export function DocumentSourcePicker({ {row.state !== "new" && !run ? ( - {row.state === "imported" - ? t("documents.sourcePicker.stateImported") - : t("documents.sourcePicker.stateDeleted")} + {heldBadgeLabel(t, row)} ) : null} @@ -678,41 +680,6 @@ export function DocumentSourcePicker({ ); } -/** How the run went, in the shared outcome vocabulary. */ -function runOutcome(counts: { - imported: number; - present: number; - failed: number; -}): WrittenOutcome { - const held = counts.imported + counts.present; - if (counts.failed === 0) return held > 0 ? "success" : "empty"; - return held > 0 ? "partial" : "failed"; -} - -/** A row's result in words, or null while it has none yet. */ -function runLabel( - t: (key: string, params?: Record) => string, - run: RowRun, -): string | null { - switch (run.state) { - case "waiting": - case "working": - return null; - case "failed": - return run.message; - case "deleted": - return t("documents.sourcePicker.result.deleted"); - case "imported": - return run.linked - ? t("documents.sourcePicker.result.importedLinked") - : t("documents.sourcePicker.result.imported"); - case "duplicate": - return run.linked - ? t("documents.sourcePicker.result.duplicateLinked") - : t("documents.sourcePicker.result.duplicate"); - } -} - /** One row's progress or result, under its title. */ function RunLine({ run }: { run: RowRun }) { const { t } = useTranslations(); @@ -751,19 +718,24 @@ function RunLine({ run }: { run: RowRun }) { ); case "imported": case "duplicate": + case "restored": return ( - {run.state === "imported" + {run.state === "restored" ? run.linked - ? t("documents.sourcePicker.result.importedLinked") - : t("documents.sourcePicker.result.imported") - : run.linked - ? t("documents.sourcePicker.result.duplicateLinked") - : t("documents.sourcePicker.result.duplicate")} + ? t("documents.sourcePicker.result.restoredLinked") + : t("documents.sourcePicker.result.restored") + : run.state === "imported" + ? run.linked + ? t("documents.sourcePicker.result.importedLinked") + : t("documents.sourcePicker.result.imported") + : run.linked + ? t("documents.sourcePicker.result.duplicateLinked") + : t("documents.sourcePicker.result.duplicate")} ); } diff --git a/src/components/documents/sources/picker-rows.ts b/src/components/documents/sources/picker-rows.ts new file mode 100644 index 000000000..eb06e24f8 --- /dev/null +++ b/src/components/documents/sources/picker-rows.ts @@ -0,0 +1,91 @@ +/** + * The document picker's row logic, apart from its rendering: which rows can be + * ticked, which ask for a deleted document back, what a held row's badge and + * a finished row's result say, and how a whole run went. + */ +import type { + DocumentImportOutcome, + DocumentSourceResultDto, +} from "@/lib/documents/sources/types"; +import type { WrittenOutcome } from "@/lib/outcome/written-outcome"; + +export type RowRun = + | { state: "waiting" } + | { state: "working" } + | { state: DocumentImportOutcome; linked: boolean } + | { state: "failed"; message: string }; + +/** + * Whether a result row can be ticked: a new one always; one deleted here too, + * since ticking it asks for it back (`revive`); an already imported one only + * when the pick does something with it: link it, or hand it to the form. + */ +export function isPickable( + row: DocumentSourceResultDto, + canTakeImported: boolean, +): boolean { + if (row.state === "imported") return canTakeImported; + return true; +} + +/** The rows the vault holds as deleted: ticking one asks for it back. */ +export function deletedSourceIds(rows: DocumentSourceResultDto[]): Set { + return new Set( + rows.filter((r) => r.state === "deleted").map((r) => r.sourceId), + ); +} + +/** + * The badge on a row the vault already holds. A deleted one still in Recently + * deleted (the search names it) says so; one removed for good says deleted. + */ +export function heldBadgeLabel( + t: (key: string) => string, + row: DocumentSourceResultDto, +): string { + if (row.state === "imported") { + return t("documents.sourcePicker.stateImported"); + } + return row.documentId + ? t("documents.sourcePicker.stateInTrash") + : t("documents.sourcePicker.stateDeleted"); +} + +/** How the run went, in the shared outcome vocabulary. */ +export function runOutcome(counts: { + imported: number; + present: number; + failed: number; +}): WrittenOutcome { + const held = counts.imported + counts.present; + if (counts.failed === 0) return held > 0 ? "success" : "empty"; + return held > 0 ? "partial" : "failed"; +} + +/** A row's result in words, or null while it has none yet. */ +export function runLabel( + t: (key: string, params?: Record) => string, + run: RowRun, +): string | null { + switch (run.state) { + case "waiting": + case "working": + return null; + case "failed": + return run.message; + case "deleted": + return t("documents.sourcePicker.result.deleted"); + case "restored": + return run.linked + ? t("documents.sourcePicker.result.restoredLinked") + : t("documents.sourcePicker.result.restored"); + case "imported": + return run.linked + ? t("documents.sourcePicker.result.importedLinked") + : t("documents.sourcePicker.result.imported"); + case "duplicate": + return run.linked + ? t("documents.sourcePicker.result.duplicateLinked") + : t("documents.sourcePicker.result.duplicate"); + } +} diff --git a/src/lib/documents/source-key.ts b/src/lib/documents/source-key.ts index f089aa3b4..a2a5d8570 100644 --- a/src/lib/documents/source-key.ts +++ b/src/lib/documents/source-key.ts @@ -88,6 +88,26 @@ export async function findSourceKey( return purged ? { state: "deleted", id: null } : null; } +/** + * Forget the ledger's record that the person deleted the document a key named + * (`DocumentImportKey`, written by the purge), matching instances the way + * {@link findSourceKey} does. Only the person's own explicit "bring it back" + * from the document picker calls this, and only once the document is in the + * vault again; an importer's upload never does, so a deletion keeps holding + * against every automatic re-send. + */ +export async function forgetDeletedSourceKey( + userId: string, + sourceSystem: DocumentSourceSystemValue, + sourceId: string, + sourceInstance: string | null = null, +): Promise { + const { count } = await prisma.documentImportKey.deleteMany({ + where: { userId, sourceSystem, sourceId, ...instanceMatch(sourceInstance) }, + }); + return count; +} + /** * How many further keys one document may collect. Real archives hold a file * under one or two ids; the cap stops a leaked token from growing the table @@ -142,9 +162,14 @@ export function checkSourceLookupRateLimit( ); } -/** What the vault holds for one source key, as the picker lists it. */ +/** + * What the vault holds for one source key, as the picker lists it. A deleted + * key names its document while that is still in Recently deleted (restorable) + * and null once the purge removed it. + */ export type SourceKeyState = - { state: "imported"; documentId: string } | { state: "deleted" }; + | { state: "imported"; documentId: string } + | { state: "deleted"; documentId: string | null }; /** * {@link findSourceKey} for a page of results at once: the same three places, @@ -205,12 +230,14 @@ export async function findSourceKeyStates( // Lowest precedence first, so the document's own key wins, as it does in // `findSourceKey`. - for (const row of purged) states.set(row.sourceId, { state: "deleted" }); + for (const row of purged) { + states.set(row.sourceId, { state: "deleted", documentId: null }); + } for (const row of wildcardFirst(aliases)) { states.set( row.sourceId, row.document.deletedAt - ? { state: "deleted" } + ? { state: "deleted", documentId: row.document.id } : { state: "imported", documentId: row.document.id }, ); } @@ -219,7 +246,7 @@ export async function findSourceKeyStates( states.set( row.sourceId, row.deletedAt - ? { state: "deleted" } + ? { state: "deleted", documentId: row.id } : { state: "imported", documentId: row.id }, ); } diff --git a/src/lib/documents/sources/types.ts b/src/lib/documents/sources/types.ts index 3fdbfd1aa..cf5eac095 100644 --- a/src/lib/documents/sources/types.ts +++ b/src/lib/documents/sources/types.ts @@ -96,7 +96,11 @@ export interface DocumentSourceResultDto { tags: string[]; sizeBytes: number | null; state: DocumentPickerResultState; - /** The vault document when `state` is `imported`. */ + /** + * The vault document when `state` is `imported`. When `state` is `deleted`, + * the document while it is still in Recently deleted, and null once it has + * been removed for good. + */ documentId: string | null; } @@ -106,7 +110,12 @@ export interface DocumentSourceSearchDto { hasMore: boolean; } -export type DocumentImportOutcome = "imported" | "duplicate" | "deleted"; +/** + * `restored` answers only an import that asked to bring a deleted document + * back (`revive`): the one still in Recently deleted was restored. + */ +export type DocumentImportOutcome = + "imported" | "duplicate" | "deleted" | "restored"; export interface DocumentSourceImportDto { outcome: DocumentImportOutcome; diff --git a/src/lib/openapi/routes/document-sources.ts b/src/lib/openapi/routes/document-sources.ts index 0b97d283c..0b71ce532 100644 --- a/src/lib/openapi/routes/document-sources.ts +++ b/src/lib/openapi/routes/document-sources.ts @@ -67,7 +67,12 @@ const resultSchema = z tags: z.array(z.string()), sizeBytes: z.number().int().nullable(), state: z.enum(["new", "imported", "deleted"]), - documentId: z.string().nullable(), + documentId: z + .string() + .nullable() + .describe( + "The vault document for `imported`. For `deleted`, the document while it is still in Recently deleted (restorable), null once it has been removed for good.", + ), }) .meta({ id: "DocumentSourceResult" }); @@ -81,7 +86,11 @@ const searchSchema = z const importResultSchema = z .object({ - outcome: z.enum(["imported", "duplicate", "deleted"]), + outcome: z + .enum(["imported", "duplicate", "deleted", "restored"]) + .describe( + "`restored` only answers a request that sent `revive: true`: the deleted document was still in Recently deleted and is back in the vault.", + ), documentId: z.string().nullable(), linked: z.boolean(), }) @@ -335,7 +344,7 @@ export const documentSourcePaths: NonNullable = { tags: ["Documents"], summary: "Import one picked document", description: - "`sourceId` must be an id the system issues (Paperless-ngx: digits; Papra: letters, digits, `_`, `-`), else 422 before anything is fetched. Fetches one document from the source and stores it in the vault through the same path as an upload, with `sourceSystem` / `sourceId` / `sourceInstance` (the connection's origin) provenance; the same id from another instance is another document. One import per key runs at a time; a concurrent second one waits and then answers `duplicate`. The key check counts toward the lookup allowance, the fetch toward the search allowance, and only a file that arrived toward the upload allowance. A source key the vault already holds is answered without a download: `duplicate` for a live document (same id), `deleted` for one the person deleted in HealthLog, which is never stored again. Bytes already in the vault under another key are `duplicate` too, and the key is remembered for that document. `link` files the document against the record the picker was opened from (a condition, a visit or a vaccination, which must be the caller's own and live; checked before anything is fetched), including a duplicate. `kind` sets the document type (default OTHER). Automatic AI reading follows the person's own setting. Counts against the person's upload allowance (60 an hour) and storage quota. 201 for a new document, 200 otherwise. Cookie session only; not delegable.", + "`sourceId` must be an id the system issues (Paperless-ngx: digits; Papra: letters, digits, `_`, `-`), else 422 before anything is fetched. Fetches one document from the source and stores it in the vault through the same path as an upload, with `sourceSystem` / `sourceId` / `sourceInstance` (the connection's origin) provenance; the same id from another instance is another document. One import per key runs at a time; a concurrent second one waits and then answers `duplicate`. The key check counts toward the lookup allowance, the fetch toward the search allowance, and only a file that arrived toward the upload allowance. A source key the vault already holds is answered without a download: `duplicate` for a live document (same id), `deleted` for one the person deleted in HealthLog, which is not stored again unless the request sends `revive: true`: then a document still in Recently deleted is restored (`restored`), and one already removed for good is fetched and stored again, its key leaving the import ledger. A token upload (`POST /api/documents/inbound`) has no such switch, so an importer never brings a deleted document back. Bytes already in the vault under another key are `duplicate` too, and the key is remembered for that document. `link` files the document against the record the picker was opened from (a condition, a visit or a vaccination, which must be the caller's own and live; checked before anything is fetched), including a duplicate. `kind` sets the document type (default OTHER). Automatic AI reading follows the person's own setting. Counts against the person's upload allowance (60 an hour) and storage quota. 201 for a new document, 200 otherwise. Cookie session only; not delegable.", security: cookieOnly, parameters: [systemParameter], requestBody: { @@ -358,7 +367,7 @@ export const documentSourcePaths: NonNullable = { }, "200": { description: - "Already held (`duplicate`) or deleted in HealthLog (`deleted`).", + "Already held (`duplicate`), deleted in HealthLog (`deleted`), or brought back from Recently deleted (`restored`).", content: { "application/json": { schema: dataEnvelope( diff --git a/src/lib/validations/document-sources.ts b/src/lib/validations/document-sources.ts index 4034d4c39..24bbaae33 100644 --- a/src/lib/validations/document-sources.ts +++ b/src/lib/validations/document-sources.ts @@ -84,6 +84,14 @@ export const documentSourceImportSchema = z.object({ id: z.string().trim().min(1).max(64), }) .optional(), + /** + * The person chose to bring back a document they deleted in HealthLog. One + * still in Recently deleted is restored (`restored`); one already removed + * for good is fetched and stored again, and the import ledger forgets its + * key. Without it a deleted document is answered `deleted` and left alone, + * as an importer's upload always is. + */ + revive: z.boolean().optional(), }); export type DocumentSourceImportInput = z.infer< typeof documentSourceImportSchema diff --git a/tests/integration/document-sources.test.ts b/tests/integration/document-sources.test.ts index 70776ef58..7793fab8d 100644 --- a/tests/integration/document-sources.test.ts +++ b/tests/integration/document-sources.test.ts @@ -522,9 +522,10 @@ describe("search and import", () => { where: { id: receipt.documentId }, data: { deletedAt: new Date() }, }); + // Deleted but still in Recently deleted: the search names the document. expect( (await (await search("q=blood")).json()).data.results[0], - ).toMatchObject({ state: "deleted", documentId: null }); + ).toMatchObject({ state: "deleted", documentId: receipt.documentId }); const revived = await importDoc({ sourceId: "41" }); expect(revived.status).toBe(200); expect((await revived.json()).data.outcome).toBe("deleted"); @@ -532,6 +533,90 @@ describe("search and import", () => { expect(await prisma.inboundDocument.count()).toBe(1); }); + it("brings a deleted document back only when asked: restored from Recently deleted, or imported again after the purge", async () => { + await seedOwner(); + await connect(); + const prisma = getPrismaClient(); + + const first = await importDoc({ sourceId: "41" }); + const id = (await first.json()).data.documentId as string; + await prisma.inboundDocument.update({ + where: { id }, + data: { deletedAt: new Date(), status: "DISCARDED" }, + }); + + // Still in Recently deleted: `revive` restores the same document, and + // nothing is downloaded. + const restored = await importDoc({ sourceId: "41", revive: true }); + expect(restored.status).toBe(200); + expect((await restored.json()).data).toEqual({ + outcome: "restored", + documentId: id, + linked: false, + }); + expect(downloads()).toBe(1); + expect( + (await prisma.inboundDocument.findUniqueOrThrow({ where: { id } })) + .deletedAt, + ).toBeNull(); + expect( + await prisma.auditLog.count({ + where: { userId, action: "documents.inbound.restore" }, + }), + ).toBe(1); + + // Deleted again and purged: the key is in the ledger, and an import that + // does not ask is still answered "deleted". + await prisma.inboundDocument.update({ + where: { id }, + data: { deletedAt: new Date(Date.now() - 31 * 86_400_000) }, + }); + const { purgeExpiredDocumentTombstones } = + await import("@/lib/jobs/document-purge"); + expect(await purgeExpiredDocumentTombstones(prisma as never)).toBe(1); + expect( + (await (await search("q=blood")).json()).data.results[0], + ).toMatchObject({ state: "deleted", documentId: null }); + const left = await importDoc({ sourceId: "41" }); + expect((await left.json()).data).toEqual({ + outcome: "deleted", + documentId: null, + linked: false, + }); + expect(downloads()).toBe(1); + + // Asked: fetched and stored again, and the ledger forgets the key. + const again = await importDoc({ sourceId: "41", revive: true }); + expect(again.status).toBe(201); + const back = (await again.json()).data; + expect(back).toMatchObject({ outcome: "imported", linked: false }); + expect(back.documentId).not.toBe(id); + expect(downloads()).toBe(2); + expect(await prisma.documentImportKey.count({ where: { userId } })).toBe(0); + expect( + (await (await search("q=blood")).json()).data.results[0], + ).toMatchObject({ state: "imported", documentId: back.documentId }); + }); + + it("keeps the deletion when a revived import cannot fetch the file", async () => { + await seedOwner(); + await connect(); + const prisma = getPrismaClient(); + await prisma.documentImportKey.create({ + data: { + userId, + sourceSystem: "PAPERLESS", + sourceId: "42", + sourceInstance: ORIGIN, + }, + }); + fake.downloadStatus = 500; + const failed = await importDoc({ sourceId: "42", revive: true }); + expect(failed.status).toBeGreaterThanOrEqual(400); + expect(await prisma.documentImportKey.count({ where: { userId } })).toBe(1); + expect(await prisma.inboundDocument.count()).toBe(0); + }); + it("answers a file already uploaded by hand as a duplicate and remembers the key", async () => { await seedOwner(); await connect();