diff --git a/messages/de.json b/messages/de.json index 16d99bc5d..add8370ee 100644 --- a/messages/de.json +++ b/messages/de.json @@ -9068,7 +9068,8 @@ "tlsPinChangedTitle": "TLS-Leaf-Zertifikat geändert: {host}", "tlsPinChangedBody": "Das von {host} ausgelieferte TLS-Leaf hat jetzt den SPKI-Pin {servedPin}, der nicht im bekannten Pin-Satz enthalten ist. Gepinnte Clients (die native App) können sich nicht mehr verbinden, sobald der alte Pin weg ist. Extrahiere den neuen Leaf-Pin erneut, ergänze ihn in TLS_LEAF_SPKI_PINS sowie im Pin-Satz des nativen Clients und veröffentliche einen Build, bevor dieses Zertifikat am {validTo} abläuft. Siehe docs/ops/tls-cert-pin.md.", "accountErasureRefusedTitle": "Konto konnte nicht gelöscht werden: {username}", - "accountErasureRefusedBody": "Die Löschfrist von {username} ist abgelaufen, aber das Konto ist die letzte aktive betreuende Person eines verwalteten Profils und wurde deshalb nicht gelöscht. Es bleibt gesperrt. Füge dem Profil eine weitere betreuende Person hinzu, lösche das Profil oder mache die Löschung unter Administration, Nutzer rückgängig." + "accountErasureRefusedBody": "Die Löschfrist von {username} ist abgelaufen, aber das Konto ist die letzte aktive betreuende Person eines verwalteten Profils und wurde deshalb nicht gelöscht. Es bleibt gesperrt. Füge dem Profil eine weitere betreuende Person hinzu, lösche das Profil oder mache die Löschung unter Administration, Nutzer rückgängig.", + "accountErasureFailedBody": "Die Löschfrist von {username} ist abgelaufen, aber das Löschen des Kontos ist fehlgeschlagen. Es bleibt gesperrt und wird jede Stunde erneut versucht, nach den anderen fälligen Konten. Den Fehler zeigt die Karte der fehlgeschlagenen Jobs unter Administration." }, "user": { "telegramTestBody": "HealthLog: Verbindung hergestellt. Telegram-Benachrichtigungen sind aktiv." diff --git a/messages/en.json b/messages/en.json index b96cfa923..0e65dbb05 100644 --- a/messages/en.json +++ b/messages/en.json @@ -9068,7 +9068,8 @@ "tlsPinChangedTitle": "TLS leaf certificate changed: {host}", "tlsPinChangedBody": "The TLS leaf served by {host} now has SPKI pin {servedPin}, which is not in the known pin set. Pinned clients (the native app) will fail to connect once the old pin is gone. Re-extract the new leaf pin, add it to TLS_LEAF_SPKI_PINS and the native client's pin set, and ship a build before this certificate expires on {validTo}. See docs/ops/tls-cert-pin.md.", "accountErasureRefusedTitle": "Account could not be erased: {username}", - "accountErasureRefusedBody": "The deletion window of {username} has ended, but the account is the last active guardian of a managed profile, so it was not erased. It stays locked. Add another guardian to the profile, delete the profile, or undo the deletion in Administration, Users." + "accountErasureRefusedBody": "The deletion window of {username} has ended, but the account is the last active guardian of a managed profile, so it was not erased. It stays locked. Add another guardian to the profile, delete the profile, or undo the deletion in Administration, Users.", + "accountErasureFailedBody": "The deletion window of {username} has ended, but erasing the account failed. It stays locked and is tried again every hour, after the other accounts due. The failing jobs card in Administration has the error." }, "user": { "telegramTestBody": "HealthLog: connection successful. Telegram notifications are active." diff --git a/messages/es.json b/messages/es.json index 3036344c7..0fd0bf0e2 100644 --- a/messages/es.json +++ b/messages/es.json @@ -9068,7 +9068,8 @@ "tlsPinChangedTitle": "Certificado de hoja TLS cambiado: {host}", "tlsPinChangedBody": "La hoja TLS servida por {host} ahora tiene el pin SPKI {servedPin}, que no está en el conjunto de pines conocidos. Los clientes con pin fijo (la app nativa) no podrán conectarse una vez que desaparezca el pin antiguo. Vuelve a extraer el nuevo pin de la hoja, añádelo a TLS_LEAF_SPKI_PINS y al conjunto de pines del cliente nativo, y publica una versión antes de que este certificado caduque el {validTo}. Consulta docs/ops/tls-cert-pin.md.", "accountErasureRefusedTitle": "No se pudo borrar la cuenta: {username}", - "accountErasureRefusedBody": "El plazo de eliminación de {username} ha terminado, pero la cuenta es la última persona responsable activa de un perfil gestionado, así que no se ha borrado. Sigue bloqueada. Añade otra persona responsable al perfil, elimina el perfil o deshaz la eliminación en Administration, Usuarios." + "accountErasureRefusedBody": "El plazo de eliminación de {username} ha terminado, pero la cuenta es la última persona responsable activa de un perfil gestionado, así que no se ha borrado. Sigue bloqueada. Añade otra persona responsable al perfil, elimina el perfil o deshaz la eliminación en Administration, Usuarios.", + "accountErasureFailedBody": "El plazo de eliminación de {username} ha terminado, pero no se pudo borrar la cuenta. Sigue bloqueada y se vuelve a intentar cada hora, después de las demás cuentas pendientes. El error aparece en la tarjeta de tareas fallidas de Administration." }, "user": { "telegramTestBody": "HealthLog: conexión correcta. Las notificaciones de Telegram están activas." diff --git a/messages/fr.json b/messages/fr.json index 59f8f5c0c..bfd69acd3 100644 --- a/messages/fr.json +++ b/messages/fr.json @@ -9068,7 +9068,8 @@ "tlsPinChangedTitle": "Certificat feuille TLS modifié : {host}", "tlsPinChangedBody": "La feuille TLS servie par {host} a maintenant le pin SPKI {servedPin}, qui ne figure pas dans l'ensemble de pins connus. Les clients épinglés (l'app native) ne pourront plus se connecter une fois l'ancien pin disparu. Réextrayez le nouveau pin de la feuille, ajoutez-le à TLS_LEAF_SPKI_PINS et à l'ensemble de pins du client natif, et publiez une version avant l'expiration de ce certificat le {validTo}. Voir docs/ops/tls-cert-pin.md.", "accountErasureRefusedTitle": "Le compte n’a pas pu être effacé : {username}", - "accountErasureRefusedBody": "Le délai de suppression de {username} est écoulé, mais ce compte est la dernière personne responsable active d’un profil géré : il n’a donc pas été effacé. Il reste verrouillé. Ajoutez une autre personne responsable au profil, supprimez le profil ou annulez la suppression dans Administration, Utilisateurs." + "accountErasureRefusedBody": "Le délai de suppression de {username} est écoulé, mais ce compte est la dernière personne responsable active d’un profil géré : il n’a donc pas été effacé. Il reste verrouillé. Ajoutez une autre personne responsable au profil, supprimez le profil ou annulez la suppression dans Administration, Utilisateurs.", + "accountErasureFailedBody": "Le délai de suppression de {username} est écoulé, mais l’effacement du compte a échoué. Il reste verrouillé et une nouvelle tentative a lieu chaque heure, après les autres comptes en attente. L’erreur figure dans la carte des tâches en échec, dans Administration." }, "user": { "telegramTestBody": "HealthLog: connexion réussie. Les notifications Telegram sont actives." diff --git a/messages/it.json b/messages/it.json index 1b93f64e4..6f6e92dce 100644 --- a/messages/it.json +++ b/messages/it.json @@ -9068,7 +9068,8 @@ "tlsPinChangedTitle": "Certificato foglia TLS modificato: {host}", "tlsPinChangedBody": "La foglia TLS servita da {host} ora ha il pin SPKI {servedPin}, che non è nell'insieme dei pin noti. I client con pin fisso (l'app nativa) non potranno più connettersi una volta scomparso il vecchio pin. Riestrai il nuovo pin della foglia, aggiungilo a TLS_LEAF_SPKI_PINS e all'insieme dei pin del client nativo, e pubblica una build prima che questo certificato scada il {validTo}. Vedi docs/ops/tls-cert-pin.md.", "accountErasureRefusedTitle": "Impossibile cancellare l’account: {username}", - "accountErasureRefusedBody": "Il periodo di eliminazione di {username} è terminato, ma l’account è l’ultima persona responsabile attiva di un profilo gestito, quindi non è stato cancellato. Resta bloccato. Aggiungi un’altra persona responsabile al profilo, elimina il profilo o annulla l’eliminazione in Administration, Utenti." + "accountErasureRefusedBody": "Il periodo di eliminazione di {username} è terminato, ma l’account è l’ultima persona responsabile attiva di un profilo gestito, quindi non è stato cancellato. Resta bloccato. Aggiungi un’altra persona responsabile al profilo, elimina il profilo o annulla l’eliminazione in Administration, Utenti.", + "accountErasureFailedBody": "Il periodo di eliminazione di {username} è terminato, ma la cancellazione dell’account non è riuscita. Resta bloccato e viene ritentata ogni ora, dopo gli altri account in scadenza. L’errore è nella scheda dei job non riusciti in Administration." }, "user": { "telegramTestBody": "HealthLog: connessione riuscita. Le notifiche Telegram sono attive." diff --git a/messages/ko.json b/messages/ko.json index 9f9e1a5da..e5932cb29 100644 --- a/messages/ko.json +++ b/messages/ko.json @@ -9068,7 +9068,8 @@ "tlsPinChangedTitle": "TLS 리프 인증서 변경: {host}", "tlsPinChangedBody": "{host}가 제공하는 TLS 리프의 SPKI 핀이 이제 {servedPin}인데, 알려진 핀 목록에 없어요. 기존 핀이 사라지면 핀을 검증하는 클라이언트(네이티브 앱)는 연결에 실패해요. 새 리프 핀을 다시 추출해 TLS_LEAF_SPKI_PINS와 네이티브 클라이언트 핀 목록에 추가하고, 이 인증서가 만료되는 {validTo} 전에 빌드를 배포하세요. docs/ops/tls-cert-pin.md를 참고하세요.", "accountErasureRefusedTitle": "계정을 지우지 못했어요: {username}", - "accountErasureRefusedBody": "{username} 계정의 삭제 유예 기간이 끝났지만, 이 계정이 관리 프로필의 마지막 활성 보호자라서 지우지 않았어요. 계정은 계속 잠겨 있어요. 프로필에 다른 보호자를 추가하거나, 프로필을 삭제하거나, 관리자의 사용자 화면에서 삭제를 취소해 주세요." + "accountErasureRefusedBody": "{username} 계정의 삭제 유예 기간이 끝났지만, 이 계정이 관리 프로필의 마지막 활성 보호자라서 지우지 않았어요. 계정은 계속 잠겨 있어요. 프로필에 다른 보호자를 추가하거나, 프로필을 삭제하거나, 관리자의 사용자 화면에서 삭제를 취소해 주세요.", + "accountErasureFailedBody": "{username} 계정의 삭제 유예 기간이 끝났지만 계정을 지우지 못했어요. 계정은 계속 잠겨 있고, 삭제할 다른 계정들 다음으로 매시간 다시 시도해요. 오류는 관리자 화면의 실패한 작업 카드에서 볼 수 있어요." }, "user": { "telegramTestBody": "HealthLog: 연결에 성공했어요. Telegram 알림이 켜져 있어요." diff --git a/messages/nl.json b/messages/nl.json index 32d63e450..a1bc952a7 100644 --- a/messages/nl.json +++ b/messages/nl.json @@ -9068,7 +9068,8 @@ "tlsPinChangedTitle": "TLS-leafcertificaat gewijzigd: {host}", "tlsPinChangedBody": "Het TLS-leafcertificaat dat {host} aanbiedt, heeft nu SPKI-pin {servedPin}, die niet in de set bekende pins staat. Clients met pinning (de native app) kunnen geen verbinding meer maken zodra de oude pin verdwenen is. Haal de nieuwe leaf-pin opnieuw op, voeg hem toe aan TLS_LEAF_SPKI_PINS en aan de pinset van de native client, en breng een build uit voordat dit certificaat op {validTo} verloopt. Zie docs/ops/tls-cert-pin.md.", "accountErasureRefusedTitle": "Account kon niet worden gewist: {username}", - "accountErasureRefusedBody": "De verwijdertermijn van {username} is afgelopen, maar het account is de laatste actieve verzorger van een beheerd profiel en is daarom niet gewist. Het blijft vergrendeld. Voeg een andere verzorger toe aan het profiel, verwijder het profiel of maak de verwijdering ongedaan in Beheer, Gebruikers." + "accountErasureRefusedBody": "De verwijdertermijn van {username} is afgelopen, maar het account is de laatste actieve verzorger van een beheerd profiel en is daarom niet gewist. Het blijft vergrendeld. Voeg een andere verzorger toe aan het profiel, verwijder het profiel of maak de verwijdering ongedaan in Beheer, Gebruikers.", + "accountErasureFailedBody": "De verwijdertermijn van {username} is afgelopen, maar het wissen van het account is mislukt. Het blijft vergrendeld en wordt elk uur opnieuw geprobeerd, na de andere accounts die aan de beurt zijn. De fout staat op de kaart met mislukte taken in Beheer." }, "user": { "telegramTestBody": "HealthLog: verbinding geslaagd. Telegram-meldingen zijn actief." diff --git a/messages/pl.json b/messages/pl.json index 4d51c9924..b2e166081 100644 --- a/messages/pl.json +++ b/messages/pl.json @@ -9068,7 +9068,8 @@ "tlsPinChangedTitle": "Zmieniono certyfikat liścia TLS: {host}", "tlsPinChangedBody": "Liść TLS serwowany przez {host} ma teraz pin SPKI {servedPin}, którego nie ma w znanym zestawie pinów. Klienci z przypiętym pinem (aplikacja natywna) nie połączą się po usunięciu starego pinu. Ponownie wyodrębnij nowy pin liścia, dodaj go do TLS_LEAF_SPKI_PINS oraz do zestawu pinów klienta natywnego i opublikuj kompilację, zanim ten certyfikat wygaśnie {validTo}. Zobacz docs/ops/tls-cert-pin.md.", "accountErasureRefusedTitle": "Nie udało się usunąć konta: {username}", - "accountErasureRefusedBody": "Okres usuwania konta {username} minął, ale to konto jest ostatnią aktywną osobą opiekującą się profilem zarządzanym, więc nie zostało usunięte. Pozostaje zablokowane. Dodaj do profilu kolejną osobę opiekującą się, usuń profil albo cofnij usunięcie w Administracja, Użytkownicy." + "accountErasureRefusedBody": "Okres usuwania konta {username} minął, ale to konto jest ostatnią aktywną osobą opiekującą się profilem zarządzanym, więc nie zostało usunięte. Pozostaje zablokowane. Dodaj do profilu kolejną osobę opiekującą się, usuń profil albo cofnij usunięcie w Administracja, Użytkownicy.", + "accountErasureFailedBody": "Okres usuwania konta {username} minął, ale usunięcie konta się nie powiodło. Konto pozostaje zablokowane, a próba jest ponawiana co godzinę, po innych kontach do usunięcia. Błąd widać na karcie nieudanych zadań w Administracja." }, "user": { "telegramTestBody": "HealthLog: połączenie udane. Powiadomienia Telegram są aktywne." diff --git a/messages/pt-BR.json b/messages/pt-BR.json index 0beda2f04..78a7998f3 100644 --- a/messages/pt-BR.json +++ b/messages/pt-BR.json @@ -9068,7 +9068,8 @@ "tlsPinChangedTitle": "Certificado folha TLS alterado: {host}", "tlsPinChangedBody": "O certificado folha TLS servido por {host} agora tem o pin SPKI {servedPin}, que não está no conjunto de pins conhecidos. Clientes com pin (o aplicativo nativo) falharão ao conectar quando o pin antigo sumir. Extraia novamente o pin da nova folha, adicione-o a TLS_LEAF_SPKI_PINS e ao conjunto de pins do cliente nativo, e publique uma build antes que este certificado expire em {validTo}. Veja docs/ops/tls-cert-pin.md.", "accountErasureRefusedTitle": "Não foi possível apagar a conta: {username}", - "accountErasureRefusedBody": "O prazo de exclusão de {username} terminou, mas a conta é o último guardião ativo de um perfil gerenciado, então não foi apagada. Ela continua bloqueada. Adicione outro guardião ao perfil, exclua o perfil ou desfaça a exclusão em Administração, Usuários." + "accountErasureRefusedBody": "O prazo de exclusão de {username} terminou, mas a conta é o último guardião ativo de um perfil gerenciado, então não foi apagada. Ela continua bloqueada. Adicione outro guardião ao perfil, exclua o perfil ou desfaça a exclusão em Administração, Usuários.", + "accountErasureFailedBody": "O prazo de exclusão de {username} terminou, mas apagar a conta falhou. Ela continua bloqueada e uma nova tentativa é feita a cada hora, depois das outras contas pendentes. O erro aparece no cartão de tarefas com falha em Administração." }, "user": { "telegramTestBody": "HealthLog: conexão bem-sucedida. As notificações do Telegram estão ativas." diff --git a/prisma/schema.prisma b/prisma/schema.prisma index 2b5fd6b2c..78fd0520b 100644 --- a/prisma/schema.prisma +++ b/prisma/schema.prisma @@ -5884,7 +5884,9 @@ model OffhostBackupState { /// `readBackupPassActivity()` behind `GET /api/admin/backups`. model BackupPassAttempt { userId String @map("user_id") - /// The pass's queue name: `data-backup` or `data-backup-offhost`. + /// The pass's queue name: `data-backup`, `data-backup-offhost`, or + /// `account-deletion-purge` (the hourly erasure of accounts whose deletion + /// window ended, which takes an account whose erasure failed last). pass String /// When the pass last started on this account. startedAt DateTime @map("started_at") diff --git a/src/lib/jobs/__tests__/account-deletion-purge.test.ts b/src/lib/jobs/__tests__/account-deletion-purge.test.ts new file mode 100644 index 000000000..a0a85b8e3 --- /dev/null +++ b/src/lib/jobs/__tests__/account-deletion-purge.test.ts @@ -0,0 +1,229 @@ +/** + * One account whose erasure throws must not hold up any other erasure, the + * sweep of expired pre-wipe copies, or the off-host kick; and it goes behind + * the other due accounts next time. + * + * Mutations that turn this red: rethrow a non-Guardian error from the loop + * (b and c stay, the sweep never runs); move the sweep out of the `finally`; + * drop the attempt marks or `orderInterruptedLast` (the failing account is + * first again in the second run). + */ +import { beforeEach, describe, expect, it, vi } from "vitest"; + +type Attempt = { + userId: string; + pass: string; + startedAt: Date; + finishedAt: Date | null; +}; + +const state = vi.hoisted(() => ({ + due: [] as Array<{ id: string; managedProfileAt: Date | null }>, + poison: new Set(), + refuse: new Set(), + attempted: [] as string[], + erased: [] as string[], + preWipeSweeps: 0, + attempts: [] as Attempt[], + workerErrors: [] as unknown[], + notices: [] as Array>, + audits: [] as Array<{ action: string; userId: string }>, + clock: 0, +})); + +vi.mock("@/lib/db", () => { + const tick = () => new Date(1_000_000 + ++state.clock); + return { + prisma: { + user: { + findMany: vi.fn(async (args: { where?: { role?: string } }) => + args.where?.role ? [{ id: "admin" }] : state.due, + ), + findUnique: vi.fn(async ({ where }: { where: { id: string } }) => ({ + username: where.id, + deletionRequestedAt: new Date(0), + })), + }, + auditLog: { + count: vi.fn( + async ({ where }: { where: { userId: string; action: string } }) => + state.audits.filter( + (a) => a.userId === where.userId && a.action === where.action, + ).length, + ), + }, + backupRestoreJob: { findMany: vi.fn(async () => []) }, + dataBackup: { + deleteMany: vi.fn(async () => { + state.preWipeSweeps++; + return { count: 3 }; + }), + }, + backupPassAttempt: { + upsert: vi.fn( + async ({ create }: { create: { userId: string; pass: string } }) => { + const row = state.attempts.find( + (a) => a.userId === create.userId && a.pass === create.pass, + ); + if (row) row.startedAt = tick(); + else + state.attempts.push({ + ...create, + startedAt: tick(), + finishedAt: null, + }); + }, + ), + updateMany: vi.fn( + async ({ where }: { where: { userId: string; pass: string } }) => { + for (const a of state.attempts) { + if (a.userId === where.userId && a.pass === where.pass) { + a.finishedAt = tick(); + } + } + }, + ), + findMany: vi.fn(async ({ where }: { where: { pass: string } }) => + state.attempts.filter((a) => a.pass === where.pass), + ), + }, + $transaction: vi.fn(), + }, + }; +}); +vi.mock("@/lib/managed-profiles/lifecycle", () => { + class LastManagedGuardianError extends Error {} + return { + ACCOUNT_DELETION_TRANSACTION_OPTIONS: {}, + LastManagedGuardianError, + withManagedProfileLock: vi.fn(), + deleteGuardianAccountWithLifecycle: vi.fn( + async (id: string, fn: (tx: unknown) => Promise) => { + state.attempted.push(id); + if (state.poison.has(id)) { + throw new Error( + "Transaction already closed: timeout 120000ms exceeded", + ); + } + if (state.refuse.has(id)) throw new LastManagedGuardianError(); + await fn({}); + }, + ), + }; +}); +vi.mock("@/lib/account-deletion/lifecycle", () => ({ + lockPendingAccount: vi.fn(async () => ({ + pending: true, + purgeAt: new Date(0), + })), + eraseAccount: vi.fn(async (_tx: unknown, id: string) => { + state.erased.push(id); + }), +})); +vi.mock("@/lib/jobs/offhost-purge", () => ({ kickOffhostPurge: vi.fn() })); +vi.mock("@/lib/auth/audit", () => ({ + auditLog: vi.fn(async (action: string, opts: { userId: string }) => { + state.audits.push({ action, userId: opts.userId }); + }), +})); +vi.mock("@/lib/notifications/dispatch-localised", () => ({ + dispatchLocalisedNotification: vi.fn(async (o: Record) => { + state.notices.push(o); + }), +})); +vi.mock("@/lib/jobs/report-worker-error", () => ({ + reportWorkerError: vi.fn(async (_q: string, err: unknown) => { + state.workerErrors.push(err); + }), +})); +vi.mock("@/lib/logging/background", () => ({ + withBackgroundEvent: (_n: string, f: (e: unknown) => unknown) => + f({ addMeta() {}, addWarning() {} }), +})); + +const { processDueAccountDeletions, handleAccountDeletionPurge } = + await import("@/lib/jobs/account-deletion-purge"); +const { kickOffhostPurge } = await import("@/lib/jobs/offhost-purge"); + +const account = (id: string) => ({ id, managedProfileAt: null }); + +beforeEach(() => { + state.due = [account("poison"), account("b"), account("c")]; + state.poison = new Set(["poison"]); + state.refuse = new Set(); + state.attempted = []; + state.erased = []; + state.preWipeSweeps = 0; + state.attempts = []; + state.workerErrors = []; + state.notices = []; + state.audits = []; + vi.mocked(kickOffhostPurge).mockClear(); +}); + +describe("the account purge with one erasure that keeps failing", () => { + it("erases every other due account, sweeps pre-wipe copies and kicks the off-host purge", async () => { + const report = await processDueAccountDeletions(new Date()); + expect(state.erased).toEqual(["b", "c"]); + expect(report).toMatchObject({ purged: 2, failed: 1, refused: 0 }); + expect(state.preWipeSweeps).toBe(1); + expect(kickOffhostPurge).toHaveBeenCalledTimes(1); + }); + + it("takes the failing account after the others in the next run", async () => { + await processDueAccountDeletions(new Date()); + state.due = [account("poison"), account("d")]; + state.attempted = []; + await processDueAccountDeletions(new Date()); + expect(state.attempted).toEqual(["d", "poison"]); + }); + + it("puts refused accounts behind the others too, longest waiting first", async () => { + state.poison = new Set(); + state.refuse = new Set(["r1", "r2"]); + state.due = [account("r1"), account("r2")]; + await processDueAccountDeletions(new Date()); + state.due = [account("r1"), account("r2"), account("e")]; + state.attempted = []; + await processDueAccountDeletions(new Date()); + expect(state.attempted).toEqual(["e", "r1", "r2"]); + }); + + it("sweeps pre-wipe copies even when the due accounts cannot be read", async () => { + const { prisma } = await import("@/lib/db"); + vi.mocked(prisma.user.findMany).mockRejectedValueOnce( + new Error("connection reset"), + ); + await expect(processDueAccountDeletions(new Date())).rejects.toThrow( + /connection reset/, + ); + expect(state.preWipeSweeps).toBe(1); + }); + + it("stops starting accounts when its budget is spent, and says how many it left", async () => { + let calls = 0; + const report = await processDueAccountDeletions(new Date(), { + shouldStop: () => ++calls > 1, + }); + expect(state.attempted).toEqual(["poison"]); + expect(report.deferred).toBe(2); + expect(state.preWipeSweeps).toBe(1); + }); + + it("reports the failure through the worker error path and tells the admins once", async () => { + const outcome = await handleAccountDeletionPurge([]); + expect(outcome).toMatchObject({ ok: true }); + expect(String(state.workerErrors[0])).toMatch( + /1 account erasure\(s\) failed.*timeout 120000ms/, + ); + expect(state.notices).toEqual([ + expect.objectContaining({ + userId: "admin", + messageKey: "notifications.admin.accountErasureFailedBody", + params: { username: "poison" }, + }), + ]); + await handleAccountDeletionPurge([]); + expect(state.notices).toHaveLength(1); + }); +}); diff --git a/src/lib/jobs/account-deletion-purge.ts b/src/lib/jobs/account-deletion-purge.ts index 32b5cf3f2..7a6b7fab3 100644 --- a/src/lib/jobs/account-deletion-purge.ts +++ b/src/lib/jobs/account-deletion-purge.ts @@ -18,10 +18,28 @@ * window would leave a managed profile nobody can reach, so that erasure is * refused, the account stays locked and pending, the run reports it, and * every admin is told once through the localized admin notification. + * + * One account never holds up the others. An erasure that throws (a record + * too large for the transaction's time limit, a lock a restore holds) is + * caught, counted, reported through the worker error path and to the admins + * once, and the run goes on with the next account. Every account the run + * starts is marked (`backup_pass_attempts`, pass `account-deletion-purge`) + * and the mark is closed only when the account is erased or no longer due, + * so an account that failed or was refused is taken after every other due + * account in the following runs, the longest-waiting first. The run stops + * starting accounts once its time budget is spent; the sweep of expired + * pre-wipe copies and the off-host kick run whatever happened before them. */ import type { Job } from "pg-boss"; import { prisma } from "@/lib/db"; +import { + markBackupAttemptFinished, + markBackupAttemptStarted, + orderInterruptedLast, + readInterruptedBackupAttempts, +} from "@/lib/jobs/backup-pass-attempts"; +import { jobBudget } from "@/lib/jobs/job-budget"; import { jobDone, jobFailed, type JobOutcome } from "@/lib/jobs/job-outcome"; import { kickOffhostPurge } from "@/lib/jobs/offhost-purge"; import { reportWorkerError } from "@/lib/jobs/report-worker-error"; @@ -43,14 +61,26 @@ import { dispatchLocalisedNotification } from "@/lib/notifications/dispatch-loca import { PRE_WIPE_BACKUP_TYPE_PREFIX } from "@/lib/account-deletion/pre-wipe-type"; import { ACCOUNT_DELETION_PURGE_QUEUE } from "./account-deletion-purge-queue"; -/** Accounts one run erases; the next run takes the rest. */ -const ACCOUNTS_PER_RUN = 50; +/** + * Due accounts one run reads. The run erases as many as its time budget + * allows; the next run takes the rest. + */ +const ACCOUNTS_PER_RUN = 200; + +/** The attempt ledger's key for this job. */ +const PASS = ACCOUNT_DELETION_PURGE_QUEUE; interface AccountDeletionPurgeReport { due: number; purged: number; /** Erasures refused by the Guardian invariant; the accounts stay pending. */ refused: number; + /** Erasures that threw; the accounts stay pending and are tried again. */ + failed: number; + /** The first error of the failed erasures, for the worker error report. */ + firstFailure: unknown; + /** Due accounts the run left for the next one when its budget ran out. */ + deferred: number; preWipeCopiesDeleted: number; } @@ -132,18 +162,33 @@ async function deleteExpiredPreWipeCopies(now: Date): Promise { return count; } -/** The audit action that marks a refused erasure as already reported. */ -const REFUSED_ERASURE_ACTION = "user.account.deletion_refused"; +/** + * Why an erasure did not happen, the audit action that marks it as already + * reported, and the notice the admins get. + */ +const NOT_ERASED = { + last_managed_guardian: { + action: "user.account.deletion_refused", + messageKey: "notifications.admin.accountErasureRefusedBody", + }, + erasure_failed: { + action: "user.account.erasure_failed", + messageKey: "notifications.admin.accountErasureFailedBody", + }, +} as const; /** * Tell every admin, once, that an account whose window ended could not be * erased, and why. The run retries the account every hour; the audit row on - * the account (`user.account.deletion_refused`, newer than the deletion - * request) is what keeps the second hour from repeating the message. A - * deletion undone and asked for again is a new request and is reported - * afresh. + * the account (one action per reason, newer than the deletion request) is + * what keeps the second hour from repeating the message. A deletion undone + * and asked for again is a new request and is reported afresh. */ -async function notifyAdminsOfRefusedErasure(accountId: string): Promise { +async function notifyAdminsOfUnerasedAccount( + accountId: string, + reason: keyof typeof NOT_ERASED, +): Promise { + const { action, messageKey } = NOT_ERASED[reason]; const account = await prisma.user.findUnique({ where: { id: accountId }, select: { username: true, deletionRequestedAt: true }, @@ -152,15 +197,15 @@ async function notifyAdminsOfRefusedErasure(accountId: string): Promise { const reported = await prisma.auditLog.count({ where: { userId: accountId, - action: REFUSED_ERASURE_ACTION, + action, createdAt: { gte: account.deletionRequestedAt }, }, }); if (reported > 0) return; - await auditLog(REFUSED_ERASURE_ACTION, { + await auditLog(action, { userId: accountId, actorUserId: null, - details: { reason: "last_managed_guardian" }, + details: { reason }, }); const admins = await prisma.user.findMany({ where: activeAccount({ role: "ADMIN" }), @@ -170,65 +215,118 @@ async function notifyAdminsOfRefusedErasure(accountId: string): Promise { await dispatchLocalisedNotification({ userId: admin.id, titleKey: "notifications.admin.accountErasureRefusedTitle", - messageKey: "notifications.admin.accountErasureRefusedBody", + messageKey, params: { username: account.username }, metadata: { source: "account-deletion-purge", url: "/admin/users" }, }); } } +/** A notice that could not be sent must not stop the run. */ +async function notifyQuietly( + accountId: string, + reason: keyof typeof NOT_ERASED, +): Promise { + try { + await notifyAdminsOfUnerasedAccount(accountId, reason); + } catch (err) { + await reportWorkerError(ACCOUNT_DELETION_PURGE_QUEUE, err); + } +} + export async function processDueAccountDeletions( now: Date = new Date(), + options: { shouldStop?: () => boolean } = {}, ): Promise { - // account-deletion: the purge's own query: exactly the accounts pending deletion. - const due = await prisma.user.findMany({ - where: { - deletionRequestedAt: { not: null }, - OR: [{ deletionPurgeAt: null }, { deletionPurgeAt: { lte: now } }], - }, - select: { id: true, managedProfileAt: true }, - orderBy: { deletionPurgeAt: "asc" }, - take: ACCOUNTS_PER_RUN, - }); - + const shouldStop = options.shouldStop ?? (() => false); const report: AccountDeletionPurgeReport = { - due: due.length, + due: 0, purged: 0, refused: 0, + failed: 0, + firstFailure: null, + deferred: 0, preWipeCopiesDeleted: 0, }; - for (const account of due) { - try { - if ( - await purgeAccount(account.id, account.managedProfileAt !== null, now) - ) { - report.purged++; + try { + // account-deletion: the purge's own query: exactly the accounts pending deletion. + const due = await prisma.user.findMany({ + where: { + deletionRequestedAt: { not: null }, + OR: [{ deletionPurgeAt: null }, { deletionPurgeAt: { lte: now } }], + }, + select: { id: true, managedProfileAt: true }, + orderBy: [{ deletionPurgeAt: "asc" }, { id: "asc" }], + take: ACCOUNTS_PER_RUN, + }); + report.due = due.length; + // An account whose last erasure failed, was refused or was cut off goes + // behind every other due account. + const ordered = orderInterruptedLast( + due, + await readInterruptedBackupAttempts(prisma, PASS), + ); + for (const [index, account] of ordered.entries()) { + if (shouldStop()) { + report.deferred = ordered.length - index; + break; + } + await markBackupAttemptStarted(prisma, PASS, account.id); + try { + if ( + await purgeAccount(account.id, account.managedProfileAt !== null, now) + ) { + report.purged++; + } + // Erased (the mark went with the account) or no longer due. + await markBackupAttemptFinished(prisma, PASS, account.id); + } catch (error) { + if (error instanceof LastManagedGuardianError) { + report.refused++; + await notifyQuietly(account.id, "last_managed_guardian"); + } else { + report.failed++; + report.firstFailure ??= error; + await notifyQuietly(account.id, "erasure_failed"); + } } - } catch (error) { - if (!(error instanceof LastManagedGuardianError)) throw error; - report.refused++; - await notifyAdminsOfRefusedErasure(account.id); } + } finally { + // The window of a pre-wipe copy ends whatever happened to the accounts. + report.preWipeCopiesDeleted = await deleteExpiredPreWipeCopies(now); + // The off-host copies of every erased account, now rather than at the + // nightly slot. The requests committed with the erasures. + if (report.purged > 0) await kickOffhostPurge(); } - report.preWipeCopiesDeleted = await deleteExpiredPreWipeCopies(now); - - // The off-host copies of every erased account, now rather than at the - // nightly slot. The requests committed with the erasures. - if (report.purged > 0) await kickOffhostPurge(); return report; } export async function handleAccountDeletionPurge( jobs: Job[], ): Promise { - void jobs; return withBackgroundEvent("job.account_deletion_purge", async (evt) => { try { - const report = await processDueAccountDeletions(); + const report = await processDueAccountDeletions(new Date(), { + shouldStop: jobBudget(jobs), + }); evt.addMeta("account_deletions_due", report.due); evt.addMeta("account_deletions_purged", report.purged); evt.addMeta("account_deletions_refused", report.refused); + evt.addMeta("account_deletions_failed", report.failed); + evt.addMeta("account_deletions_deferred", report.deferred); evt.addMeta("pre_wipe_copies_deleted", report.preWipeCopiesDeleted); + if (report.failed > 0) { + const cause = + report.firstFailure instanceof Error + ? report.firstFailure.message + : String(report.firstFailure); + await reportWorkerError( + ACCOUNT_DELETION_PURGE_QUEUE, + new Error( + `${report.failed} account erasure(s) failed and will be tried again next hour, after the other due accounts. The accounts stay locked and pending. First error: ${cause}`, + ), + ); + } if (report.refused > 0) { await reportWorkerError( ACCOUNT_DELETION_PURGE_QUEUE, @@ -241,6 +339,8 @@ export async function handleAccountDeletionPurge( account_deletions_due: report.due, account_deletions_purged: report.purged, account_deletions_refused: report.refused, + account_deletions_failed: report.failed, + account_deletions_deferred: report.deferred, pre_wipe_copies_deleted: report.preWipeCopiesDeleted, }); } catch (err) { diff --git a/src/lib/jobs/backup-pass-attempts.ts b/src/lib/jobs/backup-pass-attempts.ts index 4c7db51a6..0d8b181d4 100644 --- a/src/lib/jobs/backup-pass-attempts.ts +++ b/src/lib/jobs/backup-pass-attempts.ts @@ -31,8 +31,13 @@ import type { PrismaClient } from "@/generated/prisma/client"; import { logCaught } from "@/lib/logging/signal"; -/** The pass's queue name, which is also the marker's key. */ -export type BackupPass = "data-backup" | "data-backup-offhost"; +/** + * The pass's queue name, which is also the marker's key. The account-deletion + * purge walks the due accounts the same way and keeps its marks here too: an + * account whose erasure failed or was refused stays open and is taken last. + */ +export type BackupPass = + "data-backup" | "data-backup-offhost" | "account-deletion-purge"; type Db = Pick;