diff --git a/docs/ops/disaster-recovery.md b/docs/ops/disaster-recovery.md index 5f3469880..3d9f971b5 100644 --- a/docs/ops/disaster-recovery.md +++ b/docs/ops/disaster-recovery.md @@ -40,7 +40,8 @@ A dump holds every account that existed when it was taken, including one deleted since. Deleted accounts leave the dumps as they age out: after `DB_BACKUP_KEEP` nights on the server, and after your lifecycle rule in the bucket. Give that rule a prefix that covers `database/` too; the admin -backups page reads the bucket's rules and says when none does. This is the +backups page reads the bucket's rules and says when none does, and the +recovery readiness card calls it attention and tells the admins once. This is the retention period to state where you tell people that deleting their account deletes their data. diff --git a/messages/de.json b/messages/de.json index 83483b1a9..bb2624eeb 100644 --- a/messages/de.json +++ b/messages/de.json @@ -8662,7 +8662,8 @@ "offhost_copies": "Externe Kopien", "restore_drill": "Wiederherstellungsprobe", "offhost_setup": "Externe Sicherung", - "recovery_kit": "Wiederherstellungs-Kit" + "recovery_kit": "Wiederherstellungs-Kit", + "dump_retention": "Dumps im Bucket" }, "at": { "database_dump": "Neuester Dump: {date}", @@ -8678,7 +8679,8 @@ "offhost_copies": "Sicherungen öffnen", "restore_drill": "Sicherungen öffnen", "offhost_setup": "Sicherungen öffnen", - "recovery_kit": "Kit öffnen" + "recovery_kit": "Kit öffnen", + "dump_retention": "Sicherungen öffnen" }, "reason": { "database_dump": { @@ -8727,6 +8729,13 @@ "no_admin": "Es gibt noch kein Admin-Konto.", "not_confirmed": "Niemand hat eine Kopie der Schlüssel bestätigt, die eine Wiederherstellung braucht: {keys}.", "not_confirmed_grace": "Sichere eine Kopie der Schlüssel, die eine Wiederherstellung braucht: {keys}." + }, + "dump_retention": { + "not_configured": "Es ist kein externer Bucket eingerichtet.", + "no_offhost_dumps": "Es geht kein Datenbank-Dump in den Bucket.", + "missing": "Keine Lebenszyklus-Regel im Bucket gilt für database/. Dumps dort und die gelöschten Konten darin werden deshalb nie entfernt. Lege eine Regel für dieses Präfix an.", + "configured": "Der Bucket entfernt Dumps nach {days} Tagen.", + "unknown": "Der Bucket gibt nicht an, ob eine Regel alte Dumps entfernt. Prüfe, ob eine Lebenszyklus-Regel database/ erfasst." } }, "recovered": { @@ -8735,7 +8744,8 @@ "offhost_copies": "Jedes Konto hat wieder eine aktuelle externe Kopie.", "restore_drill": "Die Wiederherstellungsprobe ist wieder bestanden.", "offhost_setup": "Externe Kopien sind jetzt eingerichtet.", - "recovery_kit": "Das Wiederherstellungs-Kit ist wieder bestätigt." + "recovery_kit": "Das Wiederherstellungs-Kit ist wieder bestätigt.", + "dump_retention": "Eine Lebenszyklus-Regel im Bucket erfasst jetzt die Datenbank-Dumps." } } }, diff --git a/messages/en.json b/messages/en.json index 859bed279..f99c0d281 100644 --- a/messages/en.json +++ b/messages/en.json @@ -8662,7 +8662,8 @@ "offhost_copies": "Off-host copies", "restore_drill": "Restore drill", "offhost_setup": "Off-host backup", - "recovery_kit": "Recovery kit" + "recovery_kit": "Recovery kit", + "dump_retention": "Dumps in the bucket" }, "at": { "database_dump": "Newest dump: {date}", @@ -8678,7 +8679,8 @@ "offhost_copies": "Open backups", "restore_drill": "Open backups", "offhost_setup": "Open backups", - "recovery_kit": "Open the kit" + "recovery_kit": "Open the kit", + "dump_retention": "Open backups" }, "reason": { "database_dump": { @@ -8727,6 +8729,13 @@ "no_admin": "There is no admin account yet.", "not_confirmed": "Nobody has confirmed a copy of the keys a restore needs: {keys}.", "not_confirmed_grace": "Save a copy of the keys a restore needs: {keys}." + }, + "dump_retention": { + "not_configured": "No off-host bucket is set up.", + "no_offhost_dumps": "No database dump goes to the bucket.", + "missing": "No lifecycle rule in the bucket covers database/, so dumps there, and the deleted accounts inside them, are never removed. Add a rule for that prefix.", + "configured": "The bucket removes dumps after {days} days.", + "unknown": "The bucket does not say whether a rule removes old dumps. Check that a lifecycle rule covers database/." } }, "recovered": { @@ -8735,7 +8744,8 @@ "offhost_copies": "Every account has a recent off-host copy again.", "restore_drill": "The restore drill passed again.", "offhost_setup": "Off-host copies are set up now.", - "recovery_kit": "The recovery kit is confirmed again." + "recovery_kit": "The recovery kit is confirmed again.", + "dump_retention": "A lifecycle rule in the bucket covers the database dumps now." } } }, diff --git a/messages/es.json b/messages/es.json index dafb1a5e2..37bf14e28 100644 --- a/messages/es.json +++ b/messages/es.json @@ -8662,7 +8662,8 @@ "offhost_copies": "Copias fuera del host", "restore_drill": "Simulacro de restauración", "offhost_setup": "Copia fuera del host", - "recovery_kit": "Kit de recuperación" + "recovery_kit": "Kit de recuperación", + "dump_retention": "Volcados en el bucket" }, "at": { "database_dump": "Volcado más reciente: {date}", @@ -8678,7 +8679,8 @@ "offhost_copies": "Abrir copias de seguridad", "restore_drill": "Abrir copias de seguridad", "offhost_setup": "Abrir copias de seguridad", - "recovery_kit": "Abrir el kit" + "recovery_kit": "Abrir el kit", + "dump_retention": "Abrir copias de seguridad" }, "reason": { "database_dump": { @@ -8727,6 +8729,13 @@ "no_admin": "Todavía no hay ninguna cuenta de administrador.", "not_confirmed": "Nadie ha confirmado una copia de las claves que necesita una restauración: {keys}.", "not_confirmed_grace": "Guarda una copia de las claves que necesita una restauración: {keys}." + }, + "dump_retention": { + "not_configured": "No hay ningún bucket externo configurado.", + "no_offhost_dumps": "Ningún volcado de la base de datos va al bucket.", + "missing": "Ninguna regla de ciclo de vida del bucket cubre database/, así que los volcados de ahí, y las cuentas eliminadas que contienen, nunca se borran. Añade una regla para ese prefijo.", + "configured": "El bucket borra los volcados a los {days} días.", + "unknown": "El bucket no indica si alguna regla borra los volcados antiguos. Comprueba que una regla de ciclo de vida cubra database/." } }, "recovered": { @@ -8735,7 +8744,8 @@ "offhost_copies": "Cada cuenta vuelve a tener una copia fuera del host reciente.", "restore_drill": "El simulacro de restauración vuelve a superarse.", "offhost_setup": "Las copias fuera del host ya están configuradas.", - "recovery_kit": "El kit de recuperación vuelve a estar confirmado." + "recovery_kit": "El kit de recuperación vuelve a estar confirmado.", + "dump_retention": "Ahora una regla de ciclo de vida del bucket cubre los volcados de la base de datos." } } }, diff --git a/messages/fr.json b/messages/fr.json index 4d049e4c6..9afb8acda 100644 --- a/messages/fr.json +++ b/messages/fr.json @@ -8662,7 +8662,8 @@ "offhost_copies": "Copies hors hôte", "restore_drill": "Test de restauration", "offhost_setup": "Sauvegarde hors hôte", - "recovery_kit": "Kit de récupération" + "recovery_kit": "Kit de récupération", + "dump_retention": "Dumps dans le bucket" }, "at": { "database_dump": "Dump le plus récent : {date}", @@ -8678,7 +8679,8 @@ "offhost_copies": "Ouvrir les sauvegardes", "restore_drill": "Ouvrir les sauvegardes", "offhost_setup": "Ouvrir les sauvegardes", - "recovery_kit": "Ouvrir le kit" + "recovery_kit": "Ouvrir le kit", + "dump_retention": "Ouvrir les sauvegardes" }, "reason": { "database_dump": { @@ -8727,6 +8729,13 @@ "no_admin": "Il n'y a pas encore de compte administrateur.", "not_confirmed": "Personne n'a confirmé de copie des clés nécessaires à une restauration : {keys}.", "not_confirmed_grace": "Sauvegardez une copie des clés nécessaires à une restauration : {keys}." + }, + "dump_retention": { + "not_configured": "Aucun bucket hors site n'est configuré.", + "no_offhost_dumps": "Aucun dump de la base ne part dans le bucket.", + "missing": "Aucune règle de cycle de vie du bucket ne couvre database/ : les dumps qui s'y trouvent, et les comptes supprimés qu'ils contiennent, ne sont jamais effacés. Ajoutez une règle pour ce préfixe.", + "configured": "Le bucket efface les dumps au bout de {days} jours.", + "unknown": "Le bucket n'indique pas si une règle efface les anciens dumps. Vérifiez qu'une règle de cycle de vie couvre database/." } }, "recovered": { @@ -8735,7 +8744,8 @@ "offhost_copies": "Chaque compte a de nouveau une copie hors hôte récente.", "restore_drill": "Le test de restauration réussit de nouveau.", "offhost_setup": "Les copies hors hôte sont maintenant configurées.", - "recovery_kit": "Le kit de récupération est de nouveau confirmé." + "recovery_kit": "Le kit de récupération est de nouveau confirmé.", + "dump_retention": "Une règle de cycle de vie du bucket couvre désormais les dumps de la base." } } }, diff --git a/messages/it.json b/messages/it.json index a3fbd5a97..a67d5d50e 100644 --- a/messages/it.json +++ b/messages/it.json @@ -8662,7 +8662,8 @@ "offhost_copies": "Copie fuori host", "restore_drill": "Prova di ripristino", "offhost_setup": "Backup fuori host", - "recovery_kit": "Kit di recupero" + "recovery_kit": "Kit di recupero", + "dump_retention": "Dump nel bucket" }, "at": { "database_dump": "Dump più recente: {date}", @@ -8678,7 +8679,8 @@ "offhost_copies": "Apri i backup", "restore_drill": "Apri i backup", "offhost_setup": "Apri i backup", - "recovery_kit": "Apri il kit" + "recovery_kit": "Apri il kit", + "dump_retention": "Apri i backup" }, "reason": { "database_dump": { @@ -8727,6 +8729,13 @@ "no_admin": "Non esiste ancora un account amministratore.", "not_confirmed": "Nessuno ha confermato una copia delle chiavi necessarie per un ripristino: {keys}.", "not_confirmed_grace": "Salva una copia delle chiavi necessarie per un ripristino: {keys}." + }, + "dump_retention": { + "not_configured": "Non è configurato alcun bucket esterno.", + "no_offhost_dumps": "Nessun dump del database va nel bucket.", + "missing": "Nessuna regola del ciclo di vita del bucket copre database/, quindi i dump lì, e gli account eliminati che contengono, non vengono mai rimossi. Aggiungi una regola per quel prefisso.", + "configured": "Il bucket rimuove i dump dopo {days} giorni.", + "unknown": "Il bucket non dice se una regola rimuove i dump vecchi. Verifica che una regola di ciclo di vita copra database/." } }, "recovered": { @@ -8735,7 +8744,8 @@ "offhost_copies": "Ogni account ha di nuovo una copia fuori host recente.", "restore_drill": "La prova di ripristino è di nuovo riuscita.", "offhost_setup": "Le copie fuori host ora sono configurate.", - "recovery_kit": "Il kit di recupero è di nuovo confermato." + "recovery_kit": "Il kit di recupero è di nuovo confermato.", + "dump_retention": "Ora una regola di ciclo di vita del bucket copre i dump del database." } } }, diff --git a/messages/ko.json b/messages/ko.json index 3210d5022..1caf6faf9 100644 --- a/messages/ko.json +++ b/messages/ko.json @@ -8662,7 +8662,8 @@ "offhost_copies": "외부 보관 사본", "restore_drill": "복원 훈련", "offhost_setup": "외부 보관 백업", - "recovery_kit": "복구 키트" + "recovery_kit": "복구 키트", + "dump_retention": "버킷의 덤프" }, "at": { "database_dump": "최신 덤프: {date}", @@ -8678,7 +8679,8 @@ "offhost_copies": "백업 열기", "restore_drill": "백업 열기", "offhost_setup": "백업 열기", - "recovery_kit": "키트 열기" + "recovery_kit": "키트 열기", + "dump_retention": "백업 열기" }, "reason": { "database_dump": { @@ -8727,6 +8729,13 @@ "no_admin": "아직 관리자 계정이 없습니다.", "not_confirmed": "복원에 필요한 키의 사본을 아무도 확인하지 않았습니다: {keys}.", "not_confirmed_grace": "복원에 필요한 키의 사본을 보관하세요: {keys}." + }, + "dump_retention": { + "not_configured": "외부 버킷이 설정되지 않았습니다.", + "no_offhost_dumps": "버킷으로 가는 데이터베이스 덤프가 없습니다.", + "missing": "버킷의 어떤 수명 주기 규칙도 database/를 다루지 않아 그곳의 덤프와 그 안의 삭제된 계정이 절대 삭제되지 않습니다. 이 접두사에 대한 규칙을 추가하세요.", + "configured": "버킷은 {days}일 후에 덤프를 삭제합니다.", + "unknown": "버킷이 오래된 덤프를 삭제하는 규칙이 있는지 알려 주지 않습니다. 수명 주기 규칙이 database/를 포함하는지 확인하세요." } }, "recovered": { @@ -8735,7 +8744,8 @@ "offhost_copies": "모든 계정에 다시 최근 외부 보관 사본이 있습니다.", "restore_drill": "복원 훈련을 다시 통과했습니다.", "offhost_setup": "이제 외부 보관 사본이 설정되었습니다.", - "recovery_kit": "복구 키트가 다시 확인되었습니다." + "recovery_kit": "복구 키트가 다시 확인되었습니다.", + "dump_retention": "이제 버킷의 수명 주기 규칙이 데이터베이스 덤프를 포함합니다." } } }, diff --git a/messages/nl.json b/messages/nl.json index 94cbafe8f..01901ab56 100644 --- a/messages/nl.json +++ b/messages/nl.json @@ -8662,7 +8662,8 @@ "offhost_copies": "Externe kopieën", "restore_drill": "Hersteltest", "offhost_setup": "Externe back-up", - "recovery_kit": "Herstelkit" + "recovery_kit": "Herstelkit", + "dump_retention": "Dumps in de bucket" }, "at": { "database_dump": "Nieuwste dump: {date}", @@ -8678,7 +8679,8 @@ "offhost_copies": "Back-ups openen", "restore_drill": "Back-ups openen", "offhost_setup": "Back-ups openen", - "recovery_kit": "Kit openen" + "recovery_kit": "Kit openen", + "dump_retention": "Back-ups openen" }, "reason": { "database_dump": { @@ -8727,6 +8729,13 @@ "no_admin": "Er is nog geen beheerdersaccount.", "not_confirmed": "Niemand heeft een kopie bevestigd van de sleutels die een herstel nodig heeft: {keys}.", "not_confirmed_grace": "Bewaar een kopie van de sleutels die een herstel nodig heeft: {keys}." + }, + "dump_retention": { + "not_configured": "Er is geen externe bucket ingesteld.", + "no_offhost_dumps": "Er gaat geen databasedump naar de bucket.", + "missing": "Geen levenscyclusregel in de bucket geldt voor database/, dus dumps daar, en de verwijderde accounts erin, worden nooit verwijderd. Voeg een regel voor dat voorvoegsel toe.", + "configured": "De bucket verwijdert dumps na {days} dagen.", + "unknown": "De bucket zegt niet of een regel oude dumps verwijdert. Controleer of een levenscyclusregel database/ dekt." } }, "recovered": { @@ -8735,7 +8744,8 @@ "offhost_copies": "Elk account heeft weer een recente externe kopie.", "restore_drill": "De hersteltest is weer geslaagd.", "offhost_setup": "Externe kopieën zijn nu ingesteld.", - "recovery_kit": "De herstelkit is weer bevestigd." + "recovery_kit": "De herstelkit is weer bevestigd.", + "dump_retention": "Een levenscyclusregel in de bucket dekt nu de databasedumps." } } }, diff --git a/messages/pl.json b/messages/pl.json index 0aceb820b..c498fc660 100644 --- a/messages/pl.json +++ b/messages/pl.json @@ -8662,7 +8662,8 @@ "offhost_copies": "Kopie poza hostem", "restore_drill": "Próba przywracania", "offhost_setup": "Kopia poza hostem", - "recovery_kit": "Zestaw odzyskiwania" + "recovery_kit": "Zestaw odzyskiwania", + "dump_retention": "Zrzuty w buckecie" }, "at": { "database_dump": "Najnowszy zrzut: {date}", @@ -8678,7 +8679,8 @@ "offhost_copies": "Otwórz kopie zapasowe", "restore_drill": "Otwórz kopie zapasowe", "offhost_setup": "Otwórz kopie zapasowe", - "recovery_kit": "Otwórz zestaw" + "recovery_kit": "Otwórz zestaw", + "dump_retention": "Otwórz kopie zapasowe" }, "reason": { "database_dump": { @@ -8727,6 +8729,13 @@ "no_admin": "Nie ma jeszcze konta administratora.", "not_confirmed": "Nikt nie potwierdził kopii kluczy potrzebnych do przywrócenia: {keys}.", "not_confirmed_grace": "Zapisz kopię kluczy potrzebnych do przywrócenia: {keys}." + }, + "dump_retention": { + "not_configured": "Zewnętrzny bucket nie jest ustawiony.", + "no_offhost_dumps": "Żaden zrzut bazy nie trafia do bucketu.", + "missing": "Żadna reguła cyklu życia bucketu nie obejmuje database/, więc zrzuty tam, wraz z usuniętymi kontami w nich, nigdy nie są usuwane. Dodaj regułę dla tego prefiksu.", + "configured": "Bucket usuwa zrzuty po {days} dniach.", + "unknown": "Bucket nie podaje, czy jakaś reguła usuwa stare zrzuty. Sprawdź, czy reguła cyklu życia obejmuje database/." } }, "recovered": { @@ -8735,7 +8744,8 @@ "offhost_copies": "Każde konto znów ma aktualną kopię poza hostem.", "restore_drill": "Próba przywracania znów się powiodła.", "offhost_setup": "Kopie poza hostem są już skonfigurowane.", - "recovery_kit": "Zestaw odzyskiwania jest znów potwierdzony." + "recovery_kit": "Zestaw odzyskiwania jest znów potwierdzony.", + "dump_retention": "Reguła cyklu życia w buckecie obejmuje teraz zrzuty bazy." } } }, diff --git a/messages/pt-BR.json b/messages/pt-BR.json index f09bce53c..ab7eaf3ca 100644 --- a/messages/pt-BR.json +++ b/messages/pt-BR.json @@ -8662,7 +8662,8 @@ "offhost_copies": "Cópias fora do host", "restore_drill": "Teste de restauração", "offhost_setup": "Backup fora do host", - "recovery_kit": "Kit de recuperação" + "recovery_kit": "Kit de recuperação", + "dump_retention": "Dumps no bucket" }, "at": { "database_dump": "Dump mais recente: {date}", @@ -8678,7 +8679,8 @@ "offhost_copies": "Abrir backups", "restore_drill": "Abrir backups", "offhost_setup": "Abrir backups", - "recovery_kit": "Abrir o kit" + "recovery_kit": "Abrir o kit", + "dump_retention": "Abrir backups" }, "reason": { "database_dump": { @@ -8727,6 +8729,13 @@ "no_admin": "Ainda não há uma conta de administrador.", "not_confirmed": "Ninguém confirmou uma cópia das chaves de que uma restauração precisa: {keys}.", "not_confirmed_grace": "Guarde uma cópia das chaves de que uma restauração precisa: {keys}." + }, + "dump_retention": { + "not_configured": "Nenhum bucket externo está configurado.", + "no_offhost_dumps": "Nenhum dump do banco vai para o bucket.", + "missing": "Nenhuma regra de ciclo de vida do bucket cobre database/, então os dumps ali, e as contas excluídas dentro deles, nunca são removidos. Adicione uma regra para esse prefixo.", + "configured": "O bucket remove os dumps após {days} dias.", + "unknown": "O bucket não informa se alguma regra remove dumps antigos. Confira se uma regra de ciclo de vida cobre database/." } }, "recovered": { @@ -8735,7 +8744,8 @@ "offhost_copies": "Todas as contas têm novamente uma cópia recente fora do host.", "restore_drill": "O teste de restauração passou novamente.", "offhost_setup": "As cópias fora do host estão configuradas agora.", - "recovery_kit": "O kit de recuperação está confirmado novamente." + "recovery_kit": "O kit de recuperação está confirmado novamente.", + "dump_retention": "Agora uma regra de ciclo de vida do bucket cobre os dumps do banco." } } }, diff --git a/src/app/api/admin/recovery-readiness/route.ts b/src/app/api/admin/recovery-readiness/route.ts index 7c34286f9..a6d5d8dd2 100644 --- a/src/app/api/admin/recovery-readiness/route.ts +++ b/src/app/api/admin/recovery-readiness/route.ts @@ -4,7 +4,8 @@ * Every backup signal with its verdict, its time and where to fix it: the * database dump, the weekly copy, the off-host copies, the last restore drill * (no time window: a failed drill stays failed until a later one passes), the - * off-host setup and the recovery kit. The same judgement the hourly + * off-host setup, whether the bucket ever removes the database dumps, and the + * recovery kit. The same judgement the hourly * evaluation pushes to the admins (`src/lib/recovery/readiness-model.ts`). * Cookie-only admin. */ diff --git a/src/components/admin/recovery-readiness-card.tsx b/src/components/admin/recovery-readiness-card.tsx index 19cf4e5db..d77ae64d1 100644 --- a/src/components/admin/recovery-readiness-card.tsx +++ b/src/components/admin/recovery-readiness-card.tsx @@ -3,7 +3,8 @@ /** * `` — every backup signal on one card: the database * dump, the weekly account copies, the off-host copies, the last restore - * drill, the off-host setup and the recovery kit. Each row says its verdict, + * drill, the off-host setup, the bucket's rule for the dumps and the recovery + * kit. Each row says its verdict, * the time it is about and where to fix it. * * No time window anywhere: a failed drill stays on this card until a later diff --git a/src/lib/recovery/__tests__/readiness-model.test.ts b/src/lib/recovery/__tests__/readiness-model.test.ts index 0d5c6bcdb..b9d59405d 100644 --- a/src/lib/recovery/__tests__/readiness-model.test.ts +++ b/src/lib/recovery/__tests__/readiness-model.test.ts @@ -12,6 +12,7 @@ import type { DatabaseBackupHealth } from "@/lib/recovery/database-dump-health"; import { DUMP_FAILING_AFTER_HOURS, evaluateDatabaseDump, + evaluateDumpRetention, evaluateOffhostCopies, evaluateOffhostSetup, evaluateRecoveryKit, @@ -83,6 +84,7 @@ function inputs(over: Partial = {}): ReadinessInputs { weekly: { accounts: [{ createdAt: ago(60 * DAY), lastCopyAt: ago(DAY) }] }, offhost: { configured: true, + databaseLifecycle: { state: "configured", expirationDays: 30 }, accounts: [ { lastAttemptAt: ago(10 * HOUR), lastSuccessAt: ago(10 * HOUR) }, ], @@ -227,6 +229,76 @@ describe("database dump", () => { }); }); +describe("dump retention in the bucket", () => { + const goingOffhost = dump({ offhostConfigured: true }); + const evalRule = ( + databaseLifecycle: ReadinessInputs["offhost"]["databaseLifecycle"], + d: ReadinessInputs["dump"] = goingOffhost, + configured = true, + ) => + evaluateDumpRetention({ + dump: d, + offhost: { configured, databaseLifecycle, accounts: [] }, + }); + + it("asks for a rule when dumps go to a bucket that never removes them", () => { + expect(evalRule({ state: "missing", expirationDays: null })).toMatchObject({ + state: "attention", + reason: "missing", + fixPath: "/admin/backups", + }); + }); + + it("names the expiry when a rule covers database/, and stays quiet when the bucket will not say", () => { + expect(evalRule({ state: "configured", expirationDays: 30 })).toMatchObject( + { state: "ok", reason: "configured", params: { days: 30 } }, + ); + expect(evalRule({ state: "unknown", expirationDays: null })).toMatchObject({ + state: "ok", + reason: "unknown", + }); + expect(evalRule(null)).toMatchObject({ state: "ok", reason: "unknown" }); + }); + + it("does not apply without a bucket, or while no dump goes to it", () => { + const missing = { state: "missing", expirationDays: null } as const; + expect(evalRule(missing, goingOffhost, false)).toMatchObject({ + state: "inactive", + reason: "not_configured", + }); + for (const d of [ + null, + "unreadable" as const, + dump({ offhostConfigured: false }), + dump({ offhostConfigured: true, enabled: false }), + ]) { + expect(evalRule(missing, d)).toMatchObject({ + state: "inactive", + reason: "no_offhost_dumps", + }); + } + }); + + it("moves the overall verdict to attention on its own", () => { + const r = evaluateRecoveryReadiness( + inputs({ + dump: goingOffhost, + offhost: { + configured: true, + databaseLifecycle: { state: "missing", expirationDays: null }, + accounts: [ + { lastAttemptAt: ago(10 * HOUR), lastSuccessAt: ago(10 * HOUR) }, + ], + }, + }), + ); + expect( + r.signals.filter((x) => x.state !== "ok" && x.state !== "inactive"), + ).toEqual([expect.objectContaining({ id: "dump_retention" })]); + expect(r.state).toBe("attention"); + }); +}); + describe("weekly copy", () => { const weekly = ( ...accounts: Array<{ createdAt: Date; lastCopyAt: Date | null }> @@ -289,7 +361,11 @@ describe("off-host copies", () => { lastAttemptAt: Date | null; lastSuccessAt: Date | null; }> - ) => evaluateOffhostCopies({ now: NOW, offhost: { configured, accounts } }); + ) => + evaluateOffhostCopies({ + now: NOW, + offhost: { configured, databaseLifecycle: null, accounts }, + }); it("does not apply while no bucket is configured", () => { expect(offhost(false).state).toBe("inactive"); @@ -337,7 +413,10 @@ describe("off-host copies", () => { describe("restore drill", () => { const drill = (d: ReadinessInputs["drill"], configured = true) => - evaluateRestoreDrill({ offhost: { configured, accounts: [] }, drill: d }); + evaluateRestoreDrill({ + offhost: { configured, databaseLifecycle: null, accounts: [] }, + drill: d, + }); const run = { lastRunAt: ago(5 * DAY), lastSuccessAt: null as Date | null, @@ -386,7 +465,7 @@ describe("off-host setup", () => { evaluateOffhostSetup({ now: NOW, firstAdminAt, - offhost: { configured, accounts: [] }, + offhost: { configured, databaseLifecycle: null, accounts: [] }, }); it("asks for a bucket 7 days after the first admin exists", () => { diff --git a/src/lib/recovery/readiness-model.ts b/src/lib/recovery/readiness-model.ts index e544a2c26..ba7b1fbe8 100644 --- a/src/lib/recovery/readiness-model.ts +++ b/src/lib/recovery/readiness-model.ts @@ -7,8 +7,11 @@ * ordinary row. A failed monthly drill was on the failing-jobs card three days * in thirty. An off-host bucket that was never configured showed only on the * backups page, and the key step confirmed one key of the several a restore - * needs. This module turns them into six signals with one verdict each, which - * the readiness tile shows and the hourly evaluation pushes to every admin. + * needs. A bucket whose lifecycle rule left `database/` out kept every dump, + * and every deleted account inside it, forever, and only the backups page + * said so. This module turns them into seven signals with one verdict each, + * which the readiness tile shows and the hourly evaluation pushes to every + * admin. * * Pure: the inputs are read elsewhere (`readiness.ts`), so each threshold is a * unit test away and the tile and the push cannot disagree about a verdict. @@ -26,6 +29,7 @@ import { classifyOffhostBackup, type OffhostBackupFreshness, } from "@/lib/jobs/offhost-backup-freshness"; +import type { LifecycleVerdict } from "@/lib/jobs/offhost-lifecycle"; import type { DatabaseBackupHealth } from "@/lib/recovery/database-dump-health"; import type { KitEntry } from "@/lib/recovery/kit-rules"; @@ -50,6 +54,7 @@ export const READINESS_SIGNALS = [ "offhost_copies", "restore_drill", "offhost_setup", + "dump_retention", "recovery_kit", ] as const; export type ReadinessSignalId = (typeof READINESS_SIGNALS)[number]; @@ -69,6 +74,7 @@ export const READINESS_FIX_PATHS: Record = { offhost_copies: "/admin/backups", restore_drill: "/admin/backups", offhost_setup: "/admin/backups", + dump_retention: "/admin/backups", recovery_kit: "/admin/encryption", }; @@ -121,6 +127,12 @@ export interface ReadinessInputs { }; offhost: { configured: boolean; + /** + * The bucket's lifecycle rules for keys under `database/`, as the backups + * page reads them (`probeOffhostLifecycle`). Null when no bucket is + * configured. + */ + databaseLifecycle: LifecycleVerdict | null; accounts: ReadonlyArray<{ lastAttemptAt: Date | null; lastSuccessAt: Date | null; @@ -385,6 +397,42 @@ export function evaluateOffhostSetup( : signal(id, "ok", "not_configured_grace", graceEnds); } +/** + * Whether the bucket ever removes the database dumps. Each one holds every + * account that existed when it was taken, so a bucket with no rule for + * `database/` keeps a deleted account's data for as long as the bucket + * exists. Attention, not failing: every restore still works. A bucket that + * will not say (a credential without the permission, a target without the + * call) is not news either way. + */ +export function evaluateDumpRetention( + input: Pick, +): ReadinessSignal { + const id = "dump_retention"; + if (!input.offhost.configured) { + return signal(id, "inactive", "not_configured", null); + } + const { dump } = input; + const dumpsGoOffhost = + dump !== null && + dump !== "unreadable" && + dump.enabled && + dump.offhostConfigured; + if (!dumpsGoOffhost) { + return signal(id, "inactive", "no_offhost_dumps", null); + } + const rule = input.offhost.databaseLifecycle; + if (rule?.state === "missing") { + return signal(id, "attention", "missing", null); + } + if (rule?.state === "configured" && rule.expirationDays !== null) { + return signal(id, "ok", "configured", null, { + days: rule.expirationDays, + }); + } + return signal(id, "ok", "unknown", null); +} + export function evaluateRecoveryKit( input: Pick, ): ReadinessSignal { @@ -431,6 +479,7 @@ export function evaluateRecoveryReadiness( evaluateOffhostCopies(inputs), evaluateRestoreDrill(inputs), evaluateOffhostSetup(inputs), + evaluateDumpRetention(inputs), evaluateRecoveryKit(inputs), ]; const state = signals.reduce>( diff --git a/src/lib/recovery/readiness.ts b/src/lib/recovery/readiness.ts index 3e0c1553d..97782e0a9 100644 --- a/src/lib/recovery/readiness.ts +++ b/src/lib/recovery/readiness.ts @@ -11,12 +11,17 @@ * - the off-host copies: the per-account ledger the nightly run writes, and * the same "configured" rule the backups page uses (the environment, or a * ledger row as proof that an upload happened); + * - the bucket's lifecycle rule for `database/`: the same cached probe the + * backups page shows, asked only when a bucket is configured; * - the drill: its persisted last outcome, with no time window; * - the kit: the same assessment the kit card shows; * - the dump: whatever the dump module reports, through a provider. */ import { prisma } from "@/lib/db"; -import { offhostBackupConfigured } from "@/lib/jobs/offhost-backup"; +import { + offhostBackupConfigured, + probeOffhostLifecycle, +} from "@/lib/jobs/offhost-backup"; import { readDatabaseBackupHealth, type DatabaseBackupHealthProvider, @@ -74,6 +79,10 @@ export async function readReadinessInputs( const configured = offhostBackupConfigured() || accounts.some((account) => account.offhostBackupState !== null); + // Never throws: a bucket that cannot be asked answers `unknown`. + const databaseLifecycle = configured + ? (await probeOffhostLifecycle()).database + : null; return { now, @@ -88,6 +97,7 @@ export async function readReadinessInputs( }, offhost: { configured, + databaseLifecycle, accounts: accounts.map((account) => ({ lastAttemptAt: account.offhostBackupState?.lastAttemptAt ?? null, lastSuccessAt: account.offhostBackupState?.lastSuccessAt ?? null, diff --git a/tests/integration/recovery-readiness-dump-retention.test.ts b/tests/integration/recovery-readiness-dump-retention.test.ts new file mode 100644 index 000000000..f7a2f9c9b --- /dev/null +++ b/tests/integration/recovery-readiness-dump-retention.test.ts @@ -0,0 +1,134 @@ +/** + * The dump-retention signal through the real readiness reader against + * Postgres: the reader asks the bucket's lifecycle rules for `database/` the + * way the backups page does, only while a bucket is configured, and the model + * turns a missing rule into attention. + * + * The bucket call itself is stubbed (`probeOffhostLifecycle`); the parsing of + * the rules it returns is covered by the lifecycle unit tests. + */ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +import type { DatabaseBackupHealth } from "@/lib/database-backup/health"; +import type { OffhostLifecycle } from "@/lib/jobs/offhost-lifecycle"; + +const probe = vi.fn<() => Promise>(); + +vi.mock("@/lib/jobs/offhost-backup", async (importOriginal) => ({ + ...(await importOriginal()), + probeOffhostLifecycle: () => probe(), +})); + +vi.mock("@/lib/db-compat", () => ({ + ensureDbCompatibility: vi.fn().mockResolvedValue(undefined), +})); + +import { getPrismaClient, truncateAllTables } from "./setup"; + +const NOW = new Date(); + +/** A dump that went to the bucket an hour ago. */ +const dumpGoingOffhost = async (): Promise => ({ + enabled: true, + directory: "/var/lib/healthlog/backups", + keepNightly: 7, + localDirWritable: true, + localPersistent: true, + localReason: "volume", + offhostConfigured: true, + lastGood: { + at: new Date(NOW.getTime() - 3_600_000).toISOString(), + fileName: "healthlog-20261025T013000Z-daily.dump", + sizeBytes: 1024, + local: true, + offhost: true, + }, + lastFailure: null, +}); + +function useBucket(): void { + vi.stubEnv("BACKUP_S3_ENDPOINT", "https://bucket.invalid"); + vi.stubEnv("BACKUP_S3_BUCKET", "b"); + vi.stubEnv("BACKUP_S3_ACCESS_KEY", "a"); + vi.stubEnv("BACKUP_S3_SECRET_KEY", "s"); + vi.stubEnv("BACKUP_ENCRYPTION_KEY", "e".repeat(64)); +} + +async function retentionSignal() { + const { readRecoveryReadiness } = await import("@/lib/recovery/readiness"); + const readiness = await readRecoveryReadiness({ + now: NOW, + dumpHealth: dumpGoingOffhost, + }); + return { + overall: readiness.state, + signal: readiness.signals.find((s) => s.id === "dump_retention"), + }; +} + +beforeEach(async () => { + probe.mockReset(); + for (const name of [ + "BACKUP_S3_ENDPOINT", + "BACKUP_S3_BUCKET", + "BACKUP_S3_ACCESS_KEY", + "BACKUP_S3_SECRET_KEY", + "BACKUP_ENCRYPTION_KEY", + ]) { + vi.stubEnv(name, ""); + } + const prisma = getPrismaClient(); + await truncateAllTables(prisma); + await prisma.user.create({ + data: { + id: "user-retention-admin", + username: "retention-admin", + role: "ADMIN", + }, + }); +}); + +afterEach(() => { + vi.unstubAllEnvs(); +}); + +describe("dump retention in the readiness verdict (real Postgres)", () => { + it("calls a bucket with no rule for database/ attention", async () => { + useBucket(); + probe.mockResolvedValue({ + state: "configured", + expirationDays: 30, + database: { state: "missing", expirationDays: null }, + }); + const { signal } = await retentionSignal(); + expect(signal).toMatchObject({ + state: "attention", + reason: "missing", + fixPath: "/admin/backups", + }); + expect(probe).toHaveBeenCalledTimes(1); + }); + + it("is ok with the rule's expiry once one covers database/", async () => { + useBucket(); + probe.mockResolvedValue({ + state: "configured", + expirationDays: 30, + database: { state: "configured", expirationDays: 45 }, + }); + expect((await retentionSignal()).signal).toMatchObject({ + state: "ok", + reason: "configured", + params: { days: 45 }, + }); + }); + + it("does not ask a bucket that is not configured", async () => { + const { signal } = await retentionSignal(); + expect(signal).toMatchObject({ + state: "inactive", + reason: "not_configured", + }); + expect(probe).not.toHaveBeenCalled(); + }); +});