feat(recovery): judge the bucket's rule for database dumps in readiness

Only the backups page said when the bucket's lifecycle rules left
database/ out, so every dump, and every deleted account inside it, stayed
in the bucket forever without anyone being told. Readiness gains a
dump_retention signal that reads the same cached lifecycle probe: attention
when dumps go off-host and no rule covers database/, ok with the expiry
when one does, quiet when the bucket will not say, inactive without a
bucket or without off-host dumps. The hourly evaluation pushes it to the
admins like every other signal.
This commit is contained in:
Marc-André Bombeck committed 2026-10-25 16:00:35 +01:00
1 parent 351e7d1fe0
commit 66e92f36fe
16 files changed
+401 -36

No files matched your search

+2 -1
View File
@@ -40,7 +40,8 @@ A dump holds every account that existed when it was taken, including one
deleted since. Deleted accounts leave the dumps as they age out: after
`DB_BACKUP_KEEP` nights on the server, and after your lifecycle rule in the
bucket. Give that rule a prefix that covers `database/` too; the admin
backups page reads the bucket's rules and says when none does. This is the
backups page reads the bucket's rules and says when none does, and the
recovery readiness card calls it attention and tells the admins once. This is the
retention period to state where you tell people that deleting their account
deletes their data.
+13 -3
View File
@@ -8662,7 +8662,8 @@
"offhost_copies": "Externe Kopien",
"restore_drill": "Wiederherstellungsprobe",
"offhost_setup": "Externe Sicherung",
"recovery_kit": "Wiederherstellungs-Kit"
"recovery_kit": "Wiederherstellungs-Kit",
"dump_retention": "Dumps im Bucket"
},
"at": {
"database_dump": "Neuester Dump: {date}",
@@ -8678,7 +8679,8 @@
"offhost_copies": "Sicherungen öffnen",
"restore_drill": "Sicherungen öffnen",
"offhost_setup": "Sicherungen öffnen",
"recovery_kit": "Kit öffnen"
"recovery_kit": "Kit öffnen",
"dump_retention": "Sicherungen öffnen"
},
"reason": {
"database_dump": {
@@ -8727,6 +8729,13 @@
"no_admin": "Es gibt noch kein Admin-Konto.",
"not_confirmed": "Niemand hat eine Kopie der Schlüssel bestätigt, die eine Wiederherstellung braucht: {keys}.",
"not_confirmed_grace": "Sichere eine Kopie der Schlüssel, die eine Wiederherstellung braucht: {keys}."
},
"dump_retention": {
"not_configured": "Es ist kein externer Bucket eingerichtet.",
"no_offhost_dumps": "Es geht kein Datenbank-Dump in den Bucket.",
"missing": "Keine Lebenszyklus-Regel im Bucket gilt für database/. Dumps dort und die gelöschten Konten darin werden deshalb nie entfernt. Lege eine Regel für dieses Präfix an.",
"configured": "Der Bucket entfernt Dumps nach {days} Tagen.",
"unknown": "Der Bucket gibt nicht an, ob eine Regel alte Dumps entfernt. Prüfe, ob eine Lebenszyklus-Regel database/ erfasst."
}
},
"recovered": {
@@ -8735,7 +8744,8 @@
"offhost_copies": "Jedes Konto hat wieder eine aktuelle externe Kopie.",
"restore_drill": "Die Wiederherstellungsprobe ist wieder bestanden.",
"offhost_setup": "Externe Kopien sind jetzt eingerichtet.",
"recovery_kit": "Das Wiederherstellungs-Kit ist wieder bestätigt."
"recovery_kit": "Das Wiederherstellungs-Kit ist wieder bestätigt.",
"dump_retention": "Eine Lebenszyklus-Regel im Bucket erfasst jetzt die Datenbank-Dumps."
}
}
},
+13 -3
View File
@@ -8662,7 +8662,8 @@
"offhost_copies": "Off-host copies",
"restore_drill": "Restore drill",
"offhost_setup": "Off-host backup",
"recovery_kit": "Recovery kit"
"recovery_kit": "Recovery kit",
"dump_retention": "Dumps in the bucket"
},
"at": {
"database_dump": "Newest dump: {date}",
@@ -8678,7 +8679,8 @@
"offhost_copies": "Open backups",
"restore_drill": "Open backups",
"offhost_setup": "Open backups",
"recovery_kit": "Open the kit"
"recovery_kit": "Open the kit",
"dump_retention": "Open backups"
},
"reason": {
"database_dump": {
@@ -8727,6 +8729,13 @@
"no_admin": "There is no admin account yet.",
"not_confirmed": "Nobody has confirmed a copy of the keys a restore needs: {keys}.",
"not_confirmed_grace": "Save a copy of the keys a restore needs: {keys}."
},
"dump_retention": {
"not_configured": "No off-host bucket is set up.",
"no_offhost_dumps": "No database dump goes to the bucket.",
"missing": "No lifecycle rule in the bucket covers database/, so dumps there, and the deleted accounts inside them, are never removed. Add a rule for that prefix.",
"configured": "The bucket removes dumps after {days} days.",
"unknown": "The bucket does not say whether a rule removes old dumps. Check that a lifecycle rule covers database/."
}
},
"recovered": {
@@ -8735,7 +8744,8 @@
"offhost_copies": "Every account has a recent off-host copy again.",
"restore_drill": "The restore drill passed again.",
"offhost_setup": "Off-host copies are set up now.",
"recovery_kit": "The recovery kit is confirmed again."
"recovery_kit": "The recovery kit is confirmed again.",
"dump_retention": "A lifecycle rule in the bucket covers the database dumps now."
}
}
},
+13 -3
View File
@@ -8662,7 +8662,8 @@
"offhost_copies": "Copias fuera del host",
"restore_drill": "Simulacro de restauración",
"offhost_setup": "Copia fuera del host",
"recovery_kit": "Kit de recuperación"
"recovery_kit": "Kit de recuperación",
"dump_retention": "Volcados en el bucket"
},
"at": {
"database_dump": "Volcado más reciente: {date}",
@@ -8678,7 +8679,8 @@
"offhost_copies": "Abrir copias de seguridad",
"restore_drill": "Abrir copias de seguridad",
"offhost_setup": "Abrir copias de seguridad",
"recovery_kit": "Abrir el kit"
"recovery_kit": "Abrir el kit",
"dump_retention": "Abrir copias de seguridad"
},
"reason": {
"database_dump": {
@@ -8727,6 +8729,13 @@
"no_admin": "Todavía no hay ninguna cuenta de administrador.",
"not_confirmed": "Nadie ha confirmado una copia de las claves que necesita una restauración: {keys}.",
"not_confirmed_grace": "Guarda una copia de las claves que necesita una restauración: {keys}."
},
"dump_retention": {
"not_configured": "No hay ningún bucket externo configurado.",
"no_offhost_dumps": "Ningún volcado de la base de datos va al bucket.",
"missing": "Ninguna regla de ciclo de vida del bucket cubre database/, así que los volcados de ahí, y las cuentas eliminadas que contienen, nunca se borran. Añade una regla para ese prefijo.",
"configured": "El bucket borra los volcados a los {days} días.",
"unknown": "El bucket no indica si alguna regla borra los volcados antiguos. Comprueba que una regla de ciclo de vida cubra database/."
}
},
"recovered": {
@@ -8735,7 +8744,8 @@
"offhost_copies": "Cada cuenta vuelve a tener una copia fuera del host reciente.",
"restore_drill": "El simulacro de restauración vuelve a superarse.",
"offhost_setup": "Las copias fuera del host ya están configuradas.",
"recovery_kit": "El kit de recuperación vuelve a estar confirmado."
"recovery_kit": "El kit de recuperación vuelve a estar confirmado.",
"dump_retention": "Ahora una regla de ciclo de vida del bucket cubre los volcados de la base de datos."
}
}
},
+13 -3
View File
@@ -8662,7 +8662,8 @@
"offhost_copies": "Copies hors hôte",
"restore_drill": "Test de restauration",
"offhost_setup": "Sauvegarde hors hôte",
"recovery_kit": "Kit de récupération"
"recovery_kit": "Kit de récupération",
"dump_retention": "Dumps dans le bucket"
},
"at": {
"database_dump": "Dump le plus récent : {date}",
@@ -8678,7 +8679,8 @@
"offhost_copies": "Ouvrir les sauvegardes",
"restore_drill": "Ouvrir les sauvegardes",
"offhost_setup": "Ouvrir les sauvegardes",
"recovery_kit": "Ouvrir le kit"
"recovery_kit": "Ouvrir le kit",
"dump_retention": "Ouvrir les sauvegardes"
},
"reason": {
"database_dump": {
@@ -8727,6 +8729,13 @@
"no_admin": "Il n'y a pas encore de compte administrateur.",
"not_confirmed": "Personne n'a confirmé de copie des clés nécessaires à une restauration : {keys}.",
"not_confirmed_grace": "Sauvegardez une copie des clés nécessaires à une restauration : {keys}."
},
"dump_retention": {
"not_configured": "Aucun bucket hors site n'est configuré.",
"no_offhost_dumps": "Aucun dump de la base ne part dans le bucket.",
"missing": "Aucune règle de cycle de vie du bucket ne couvre database/ : les dumps qui s'y trouvent, et les comptes supprimés qu'ils contiennent, ne sont jamais effacés. Ajoutez une règle pour ce préfixe.",
"configured": "Le bucket efface les dumps au bout de {days} jours.",
"unknown": "Le bucket n'indique pas si une règle efface les anciens dumps. Vérifiez qu'une règle de cycle de vie couvre database/."
}
},
"recovered": {
@@ -8735,7 +8744,8 @@
"offhost_copies": "Chaque compte a de nouveau une copie hors hôte récente.",
"restore_drill": "Le test de restauration réussit de nouveau.",
"offhost_setup": "Les copies hors hôte sont maintenant configurées.",
"recovery_kit": "Le kit de récupération est de nouveau confirmé."
"recovery_kit": "Le kit de récupération est de nouveau confirmé.",
"dump_retention": "Une règle de cycle de vie du bucket couvre désormais les dumps de la base."
}
}
},
+13 -3
View File
@@ -8662,7 +8662,8 @@
"offhost_copies": "Copie fuori host",
"restore_drill": "Prova di ripristino",
"offhost_setup": "Backup fuori host",
"recovery_kit": "Kit di recupero"
"recovery_kit": "Kit di recupero",
"dump_retention": "Dump nel bucket"
},
"at": {
"database_dump": "Dump più recente: {date}",
@@ -8678,7 +8679,8 @@
"offhost_copies": "Apri i backup",
"restore_drill": "Apri i backup",
"offhost_setup": "Apri i backup",
"recovery_kit": "Apri il kit"
"recovery_kit": "Apri il kit",
"dump_retention": "Apri i backup"
},
"reason": {
"database_dump": {
@@ -8727,6 +8729,13 @@
"no_admin": "Non esiste ancora un account amministratore.",
"not_confirmed": "Nessuno ha confermato una copia delle chiavi necessarie per un ripristino: {keys}.",
"not_confirmed_grace": "Salva una copia delle chiavi necessarie per un ripristino: {keys}."
},
"dump_retention": {
"not_configured": "Non è configurato alcun bucket esterno.",
"no_offhost_dumps": "Nessun dump del database va nel bucket.",
"missing": "Nessuna regola del ciclo di vita del bucket copre database/, quindi i dump lì, e gli account eliminati che contengono, non vengono mai rimossi. Aggiungi una regola per quel prefisso.",
"configured": "Il bucket rimuove i dump dopo {days} giorni.",
"unknown": "Il bucket non dice se una regola rimuove i dump vecchi. Verifica che una regola di ciclo di vita copra database/."
}
},
"recovered": {
@@ -8735,7 +8744,8 @@
"offhost_copies": "Ogni account ha di nuovo una copia fuori host recente.",
"restore_drill": "La prova di ripristino è di nuovo riuscita.",
"offhost_setup": "Le copie fuori host ora sono configurate.",
"recovery_kit": "Il kit di recupero è di nuovo confermato."
"recovery_kit": "Il kit di recupero è di nuovo confermato.",
"dump_retention": "Ora una regola di ciclo di vita del bucket copre i dump del database."
}
}
},
+13 -3
View File
@@ -8662,7 +8662,8 @@
"offhost_copies": "외부 보관 사본",
"restore_drill": "복원 훈련",
"offhost_setup": "외부 보관 백업",
"recovery_kit": "복구 키트"
"recovery_kit": "복구 키트",
"dump_retention": "버킷의 덤프"
},
"at": {
"database_dump": "최신 덤프: {date}",
@@ -8678,7 +8679,8 @@
"offhost_copies": "백업 열기",
"restore_drill": "백업 열기",
"offhost_setup": "백업 열기",
"recovery_kit": "키트 열기"
"recovery_kit": "키트 열기",
"dump_retention": "백업 열기"
},
"reason": {
"database_dump": {
@@ -8727,6 +8729,13 @@
"no_admin": "아직 관리자 계정이 없습니다.",
"not_confirmed": "복원에 필요한 키의 사본을 아무도 확인하지 않았습니다: {keys}.",
"not_confirmed_grace": "복원에 필요한 키의 사본을 보관하세요: {keys}."
},
"dump_retention": {
"not_configured": "외부 버킷이 설정되지 않았습니다.",
"no_offhost_dumps": "버킷으로 가는 데이터베이스 덤프가 없습니다.",
"missing": "버킷의 어떤 수명 주기 규칙도 database/를 다루지 않아 그곳의 덤프와 그 안의 삭제된 계정이 절대 삭제되지 않습니다. 이 접두사에 대한 규칙을 추가하세요.",
"configured": "버킷은 {days}일 후에 덤프를 삭제합니다.",
"unknown": "버킷이 오래된 덤프를 삭제하는 규칙이 있는지 알려 주지 않습니다. 수명 주기 규칙이 database/를 포함하는지 확인하세요."
}
},
"recovered": {
@@ -8735,7 +8744,8 @@
"offhost_copies": "모든 계정에 다시 최근 외부 보관 사본이 있습니다.",
"restore_drill": "복원 훈련을 다시 통과했습니다.",
"offhost_setup": "이제 외부 보관 사본이 설정되었습니다.",
"recovery_kit": "복구 키트가 다시 확인되었습니다."
"recovery_kit": "복구 키트가 다시 확인되었습니다.",
"dump_retention": "이제 버킷의 수명 주기 규칙이 데이터베이스 덤프를 포함합니다."
}
}
},
+13 -3
View File
@@ -8662,7 +8662,8 @@
"offhost_copies": "Externe kopieën",
"restore_drill": "Hersteltest",
"offhost_setup": "Externe back-up",
"recovery_kit": "Herstelkit"
"recovery_kit": "Herstelkit",
"dump_retention": "Dumps in de bucket"
},
"at": {
"database_dump": "Nieuwste dump: {date}",
@@ -8678,7 +8679,8 @@
"offhost_copies": "Back-ups openen",
"restore_drill": "Back-ups openen",
"offhost_setup": "Back-ups openen",
"recovery_kit": "Kit openen"
"recovery_kit": "Kit openen",
"dump_retention": "Back-ups openen"
},
"reason": {
"database_dump": {
@@ -8727,6 +8729,13 @@
"no_admin": "Er is nog geen beheerdersaccount.",
"not_confirmed": "Niemand heeft een kopie bevestigd van de sleutels die een herstel nodig heeft: {keys}.",
"not_confirmed_grace": "Bewaar een kopie van de sleutels die een herstel nodig heeft: {keys}."
},
"dump_retention": {
"not_configured": "Er is geen externe bucket ingesteld.",
"no_offhost_dumps": "Er gaat geen databasedump naar de bucket.",
"missing": "Geen levenscyclusregel in de bucket geldt voor database/, dus dumps daar, en de verwijderde accounts erin, worden nooit verwijderd. Voeg een regel voor dat voorvoegsel toe.",
"configured": "De bucket verwijdert dumps na {days} dagen.",
"unknown": "De bucket zegt niet of een regel oude dumps verwijdert. Controleer of een levenscyclusregel database/ dekt."
}
},
"recovered": {
@@ -8735,7 +8744,8 @@
"offhost_copies": "Elk account heeft weer een recente externe kopie.",
"restore_drill": "De hersteltest is weer geslaagd.",
"offhost_setup": "Externe kopieën zijn nu ingesteld.",
"recovery_kit": "De herstelkit is weer bevestigd."
"recovery_kit": "De herstelkit is weer bevestigd.",
"dump_retention": "Een levenscyclusregel in de bucket dekt nu de databasedumps."
}
}
},
+13 -3
View File
@@ -8662,7 +8662,8 @@
"offhost_copies": "Kopie poza hostem",
"restore_drill": "Próba przywracania",
"offhost_setup": "Kopia poza hostem",
"recovery_kit": "Zestaw odzyskiwania"
"recovery_kit": "Zestaw odzyskiwania",
"dump_retention": "Zrzuty w buckecie"
},
"at": {
"database_dump": "Najnowszy zrzut: {date}",
@@ -8678,7 +8679,8 @@
"offhost_copies": "Otwórz kopie zapasowe",
"restore_drill": "Otwórz kopie zapasowe",
"offhost_setup": "Otwórz kopie zapasowe",
"recovery_kit": "Otwórz zestaw"
"recovery_kit": "Otwórz zestaw",
"dump_retention": "Otwórz kopie zapasowe"
},
"reason": {
"database_dump": {
@@ -8727,6 +8729,13 @@
"no_admin": "Nie ma jeszcze konta administratora.",
"not_confirmed": "Nikt nie potwierdził kopii kluczy potrzebnych do przywrócenia: {keys}.",
"not_confirmed_grace": "Zapisz kopię kluczy potrzebnych do przywrócenia: {keys}."
},
"dump_retention": {
"not_configured": "Zewnętrzny bucket nie jest ustawiony.",
"no_offhost_dumps": "Żaden zrzut bazy nie trafia do bucketu.",
"missing": "Żadna reguła cyklu życia bucketu nie obejmuje database/, więc zrzuty tam, wraz z usuniętymi kontami w nich, nigdy nie są usuwane. Dodaj regułę dla tego prefiksu.",
"configured": "Bucket usuwa zrzuty po {days} dniach.",
"unknown": "Bucket nie podaje, czy jakaś reguła usuwa stare zrzuty. Sprawdź, czy reguła cyklu życia obejmuje database/."
}
},
"recovered": {
@@ -8735,7 +8744,8 @@
"offhost_copies": "Każde konto znów ma aktualną kopię poza hostem.",
"restore_drill": "Próba przywracania znów się powiodła.",
"offhost_setup": "Kopie poza hostem są już skonfigurowane.",
"recovery_kit": "Zestaw odzyskiwania jest znów potwierdzony."
"recovery_kit": "Zestaw odzyskiwania jest znów potwierdzony.",
"dump_retention": "Reguła cyklu życia w buckecie obejmuje teraz zrzuty bazy."
}
}
},
+13 -3
View File
@@ -8662,7 +8662,8 @@
"offhost_copies": "Cópias fora do host",
"restore_drill": "Teste de restauração",
"offhost_setup": "Backup fora do host",
"recovery_kit": "Kit de recuperação"
"recovery_kit": "Kit de recuperação",
"dump_retention": "Dumps no bucket"
},
"at": {
"database_dump": "Dump mais recente: {date}",
@@ -8678,7 +8679,8 @@
"offhost_copies": "Abrir backups",
"restore_drill": "Abrir backups",
"offhost_setup": "Abrir backups",
"recovery_kit": "Abrir o kit"
"recovery_kit": "Abrir o kit",
"dump_retention": "Abrir backups"
},
"reason": {
"database_dump": {
@@ -8727,6 +8729,13 @@
"no_admin": "Ainda não há uma conta de administrador.",
"not_confirmed": "Ninguém confirmou uma cópia das chaves de que uma restauração precisa: {keys}.",
"not_confirmed_grace": "Guarde uma cópia das chaves de que uma restauração precisa: {keys}."
},
"dump_retention": {
"not_configured": "Nenhum bucket externo está configurado.",
"no_offhost_dumps": "Nenhum dump do banco vai para o bucket.",
"missing": "Nenhuma regra de ciclo de vida do bucket cobre database/, então os dumps ali, e as contas excluídas dentro deles, nunca são removidos. Adicione uma regra para esse prefixo.",
"configured": "O bucket remove os dumps após {days} dias.",
"unknown": "O bucket não informa se alguma regra remove dumps antigos. Confira se uma regra de ciclo de vida cobre database/."
}
},
"recovered": {
@@ -8735,7 +8744,8 @@
"offhost_copies": "Todas as contas têm novamente uma cópia recente fora do host.",
"restore_drill": "O teste de restauração passou novamente.",
"offhost_setup": "As cópias fora do host estão configuradas agora.",
"recovery_kit": "O kit de recuperação está confirmado novamente."
"recovery_kit": "O kit de recuperação está confirmado novamente.",
"dump_retention": "Agora uma regra de ciclo de vida do bucket cobre os dumps do banco."
}
}
},
@@ -4,7 +4,8 @@
* Every backup signal with its verdict, its time and where to fix it: the
* database dump, the weekly copy, the off-host copies, the last restore drill
* (no time window: a failed drill stays failed until a later one passes), the
* off-host setup and the recovery kit. The same judgement the hourly
* off-host setup, whether the bucket ever removes the database dumps, and the
* recovery kit. The same judgement the hourly
* evaluation pushes to the admins (`src/lib/recovery/readiness-model.ts`).
* Cookie-only admin.
*/
@@ -3,7 +3,8 @@
/**
* `<RecoveryReadinessCard>` — every backup signal on one card: the database
* dump, the weekly account copies, the off-host copies, the last restore
* drill, the off-host setup and the recovery kit. Each row says its verdict,
* drill, the off-host setup, the bucket's rule for the dumps and the recovery
* kit. Each row says its verdict,
* the time it is about and where to fix it.
*
* No time window anywhere: a failed drill stays on this card until a later
@@ -12,6 +12,7 @@ import type { DatabaseBackupHealth } from "@/lib/recovery/database-dump-health";
import {
DUMP_FAILING_AFTER_HOURS,
evaluateDatabaseDump,
evaluateDumpRetention,
evaluateOffhostCopies,
evaluateOffhostSetup,
evaluateRecoveryKit,
@@ -83,6 +84,7 @@ function inputs(over: Partial<ReadinessInputs> = {}): ReadinessInputs {
weekly: { accounts: [{ createdAt: ago(60 * DAY), lastCopyAt: ago(DAY) }] },
offhost: {
configured: true,
databaseLifecycle: { state: "configured", expirationDays: 30 },
accounts: [
{ lastAttemptAt: ago(10 * HOUR), lastSuccessAt: ago(10 * HOUR) },
],
@@ -227,6 +229,76 @@ describe("database dump", () => {
});
});
describe("dump retention in the bucket", () => {
const goingOffhost = dump({ offhostConfigured: true });
const evalRule = (
databaseLifecycle: ReadinessInputs["offhost"]["databaseLifecycle"],
d: ReadinessInputs["dump"] = goingOffhost,
configured = true,
) =>
evaluateDumpRetention({
dump: d,
offhost: { configured, databaseLifecycle, accounts: [] },
});
it("asks for a rule when dumps go to a bucket that never removes them", () => {
expect(evalRule({ state: "missing", expirationDays: null })).toMatchObject({
state: "attention",
reason: "missing",
fixPath: "/admin/backups",
});
});
it("names the expiry when a rule covers database/, and stays quiet when the bucket will not say", () => {
expect(evalRule({ state: "configured", expirationDays: 30 })).toMatchObject(
{ state: "ok", reason: "configured", params: { days: 30 } },
);
expect(evalRule({ state: "unknown", expirationDays: null })).toMatchObject({
state: "ok",
reason: "unknown",
});
expect(evalRule(null)).toMatchObject({ state: "ok", reason: "unknown" });
});
it("does not apply without a bucket, or while no dump goes to it", () => {
const missing = { state: "missing", expirationDays: null } as const;
expect(evalRule(missing, goingOffhost, false)).toMatchObject({
state: "inactive",
reason: "not_configured",
});
for (const d of [
null,
"unreadable" as const,
dump({ offhostConfigured: false }),
dump({ offhostConfigured: true, enabled: false }),
]) {
expect(evalRule(missing, d)).toMatchObject({
state: "inactive",
reason: "no_offhost_dumps",
});
}
});
it("moves the overall verdict to attention on its own", () => {
const r = evaluateRecoveryReadiness(
inputs({
dump: goingOffhost,
offhost: {
configured: true,
databaseLifecycle: { state: "missing", expirationDays: null },
accounts: [
{ lastAttemptAt: ago(10 * HOUR), lastSuccessAt: ago(10 * HOUR) },
],
},
}),
);
expect(
r.signals.filter((x) => x.state !== "ok" && x.state !== "inactive"),
).toEqual([expect.objectContaining({ id: "dump_retention" })]);
expect(r.state).toBe("attention");
});
});
describe("weekly copy", () => {
const weekly = (
...accounts: Array<{ createdAt: Date; lastCopyAt: Date | null }>
@@ -289,7 +361,11 @@ describe("off-host copies", () => {
lastAttemptAt: Date | null;
lastSuccessAt: Date | null;
}>
) => evaluateOffhostCopies({ now: NOW, offhost: { configured, accounts } });
) =>
evaluateOffhostCopies({
now: NOW,
offhost: { configured, databaseLifecycle: null, accounts },
});
it("does not apply while no bucket is configured", () => {
expect(offhost(false).state).toBe("inactive");
@@ -337,7 +413,10 @@ describe("off-host copies", () => {
describe("restore drill", () => {
const drill = (d: ReadinessInputs["drill"], configured = true) =>
evaluateRestoreDrill({ offhost: { configured, accounts: [] }, drill: d });
evaluateRestoreDrill({
offhost: { configured, databaseLifecycle: null, accounts: [] },
drill: d,
});
const run = {
lastRunAt: ago(5 * DAY),
lastSuccessAt: null as Date | null,
@@ -386,7 +465,7 @@ describe("off-host setup", () => {
evaluateOffhostSetup({
now: NOW,
firstAdminAt,
offhost: { configured, accounts: [] },
offhost: { configured, databaseLifecycle: null, accounts: [] },
});
it("asks for a bucket 7 days after the first admin exists", () => {
+51 -2
View File
@@ -7,8 +7,11 @@
* ordinary row. A failed monthly drill was on the failing-jobs card three days
* in thirty. An off-host bucket that was never configured showed only on the
* backups page, and the key step confirmed one key of the several a restore
* needs. This module turns them into six signals with one verdict each, which
* the readiness tile shows and the hourly evaluation pushes to every admin.
* needs. A bucket whose lifecycle rule left `database/` out kept every dump,
* and every deleted account inside it, forever, and only the backups page
* said so. This module turns them into seven signals with one verdict each,
* which the readiness tile shows and the hourly evaluation pushes to every
* admin.
*
* Pure: the inputs are read elsewhere (`readiness.ts`), so each threshold is a
* unit test away and the tile and the push cannot disagree about a verdict.
@@ -26,6 +29,7 @@ import {
classifyOffhostBackup,
type OffhostBackupFreshness,
} from "@/lib/jobs/offhost-backup-freshness";
import type { LifecycleVerdict } from "@/lib/jobs/offhost-lifecycle";
import type { DatabaseBackupHealth } from "@/lib/recovery/database-dump-health";
import type { KitEntry } from "@/lib/recovery/kit-rules";
@@ -50,6 +54,7 @@ export const READINESS_SIGNALS = [
"offhost_copies",
"restore_drill",
"offhost_setup",
"dump_retention",
"recovery_kit",
] as const;
export type ReadinessSignalId = (typeof READINESS_SIGNALS)[number];
@@ -69,6 +74,7 @@ export const READINESS_FIX_PATHS: Record<ReadinessSignalId, string> = {
offhost_copies: "/admin/backups",
restore_drill: "/admin/backups",
offhost_setup: "/admin/backups",
dump_retention: "/admin/backups",
recovery_kit: "/admin/encryption",
};
@@ -121,6 +127,12 @@ export interface ReadinessInputs {
};
offhost: {
configured: boolean;
/**
* The bucket's lifecycle rules for keys under `database/`, as the backups
* page reads them (`probeOffhostLifecycle`). Null when no bucket is
* configured.
*/
databaseLifecycle: LifecycleVerdict | null;
accounts: ReadonlyArray<{
lastAttemptAt: Date | null;
lastSuccessAt: Date | null;
@@ -385,6 +397,42 @@ export function evaluateOffhostSetup(
: signal(id, "ok", "not_configured_grace", graceEnds);
}
/**
* Whether the bucket ever removes the database dumps. Each one holds every
* account that existed when it was taken, so a bucket with no rule for
* `database/` keeps a deleted account's data for as long as the bucket
* exists. Attention, not failing: every restore still works. A bucket that
* will not say (a credential without the permission, a target without the
* call) is not news either way.
*/
export function evaluateDumpRetention(
input: Pick<ReadinessInputs, "dump" | "offhost">,
): ReadinessSignal {
const id = "dump_retention";
if (!input.offhost.configured) {
return signal(id, "inactive", "not_configured", null);
}
const { dump } = input;
const dumpsGoOffhost =
dump !== null &&
dump !== "unreadable" &&
dump.enabled &&
dump.offhostConfigured;
if (!dumpsGoOffhost) {
return signal(id, "inactive", "no_offhost_dumps", null);
}
const rule = input.offhost.databaseLifecycle;
if (rule?.state === "missing") {
return signal(id, "attention", "missing", null);
}
if (rule?.state === "configured" && rule.expirationDays !== null) {
return signal(id, "ok", "configured", null, {
days: rule.expirationDays,
});
}
return signal(id, "ok", "unknown", null);
}
export function evaluateRecoveryKit(
input: Pick<ReadinessInputs, "now" | "firstAdminAt" | "kit">,
): ReadinessSignal {
@@ -431,6 +479,7 @@ export function evaluateRecoveryReadiness(
evaluateOffhostCopies(inputs),
evaluateRestoreDrill(inputs),
evaluateOffhostSetup(inputs),
evaluateDumpRetention(inputs),
evaluateRecoveryKit(inputs),
];
const state = signals.reduce<Exclude<ReadinessState, "inactive">>(
+11 -1
View File
@@ -11,12 +11,17 @@
* - the off-host copies: the per-account ledger the nightly run writes, and
* the same "configured" rule the backups page uses (the environment, or a
* ledger row as proof that an upload happened);
* - the bucket's lifecycle rule for `database/`: the same cached probe the
* backups page shows, asked only when a bucket is configured;
* - the drill: its persisted last outcome, with no time window;
* - the kit: the same assessment the kit card shows;
* - the dump: whatever the dump module reports, through a provider.
*/
import { prisma } from "@/lib/db";
import { offhostBackupConfigured } from "@/lib/jobs/offhost-backup";
import {
offhostBackupConfigured,
probeOffhostLifecycle,
} from "@/lib/jobs/offhost-backup";
import {
readDatabaseBackupHealth,
type DatabaseBackupHealthProvider,
@@ -74,6 +79,10 @@ export async function readReadinessInputs(
const configured =
offhostBackupConfigured() ||
accounts.some((account) => account.offhostBackupState !== null);
// Never throws: a bucket that cannot be asked answers `unknown`.
const databaseLifecycle = configured
? (await probeOffhostLifecycle()).database
: null;
return {
now,
@@ -88,6 +97,7 @@ export async function readReadinessInputs(
},
offhost: {
configured,
databaseLifecycle,
accounts: accounts.map((account) => ({
lastAttemptAt: account.offhostBackupState?.lastAttemptAt ?? null,
lastSuccessAt: account.offhostBackupState?.lastSuccessAt ?? null,
@@ -0,0 +1,134 @@
/**
* The dump-retention signal through the real readiness reader against
* Postgres: the reader asks the bucket's lifecycle rules for `database/` the
* way the backups page does, only while a bucket is configured, and the model
* turns a missing rule into attention.
*
* The bucket call itself is stubbed (`probeOffhostLifecycle`); the parsing of
* the rules it returns is covered by the lifecycle unit tests.
*/
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import type { DatabaseBackupHealth } from "@/lib/database-backup/health";
import type { OffhostLifecycle } from "@/lib/jobs/offhost-lifecycle";
const probe = vi.fn<() => Promise<OffhostLifecycle>>();
vi.mock("@/lib/jobs/offhost-backup", async (importOriginal) => ({
...(await importOriginal<typeof import("@/lib/jobs/offhost-backup")>()),
probeOffhostLifecycle: () => probe(),
}));
vi.mock("@/lib/db-compat", () => ({
ensureDbCompatibility: vi.fn().mockResolvedValue(undefined),
}));
import { getPrismaClient, truncateAllTables } from "./setup";
const NOW = new Date();
/** A dump that went to the bucket an hour ago. */
const dumpGoingOffhost = async (): Promise<DatabaseBackupHealth> => ({
enabled: true,
directory: "/var/lib/healthlog/backups",
keepNightly: 7,
localDirWritable: true,
localPersistent: true,
localReason: "volume",
offhostConfigured: true,
lastGood: {
at: new Date(NOW.getTime() - 3_600_000).toISOString(),
fileName: "healthlog-20261025T013000Z-daily.dump",
sizeBytes: 1024,
local: true,
offhost: true,
},
lastFailure: null,
});
function useBucket(): void {
vi.stubEnv("BACKUP_S3_ENDPOINT", "https://bucket.invalid");
vi.stubEnv("BACKUP_S3_BUCKET", "b");
vi.stubEnv("BACKUP_S3_ACCESS_KEY", "a");
vi.stubEnv("BACKUP_S3_SECRET_KEY", "s");
vi.stubEnv("BACKUP_ENCRYPTION_KEY", "e".repeat(64));
}
async function retentionSignal() {
const { readRecoveryReadiness } = await import("@/lib/recovery/readiness");
const readiness = await readRecoveryReadiness({
now: NOW,
dumpHealth: dumpGoingOffhost,
});
return {
overall: readiness.state,
signal: readiness.signals.find((s) => s.id === "dump_retention"),
};
}
beforeEach(async () => {
probe.mockReset();
for (const name of [
"BACKUP_S3_ENDPOINT",
"BACKUP_S3_BUCKET",
"BACKUP_S3_ACCESS_KEY",
"BACKUP_S3_SECRET_KEY",
"BACKUP_ENCRYPTION_KEY",
]) {
vi.stubEnv(name, "");
}
const prisma = getPrismaClient();
await truncateAllTables(prisma);
await prisma.user.create({
data: {
id: "user-retention-admin",
username: "retention-admin",
role: "ADMIN",
},
});
});
afterEach(() => {
vi.unstubAllEnvs();
});
describe("dump retention in the readiness verdict (real Postgres)", () => {
it("calls a bucket with no rule for database/ attention", async () => {
useBucket();
probe.mockResolvedValue({
state: "configured",
expirationDays: 30,
database: { state: "missing", expirationDays: null },
});
const { signal } = await retentionSignal();
expect(signal).toMatchObject({
state: "attention",
reason: "missing",
fixPath: "/admin/backups",
});
expect(probe).toHaveBeenCalledTimes(1);
});
it("is ok with the rule's expiry once one covers database/", async () => {
useBucket();
probe.mockResolvedValue({
state: "configured",
expirationDays: 30,
database: { state: "configured", expirationDays: 45 },
});
expect((await retentionSignal()).signal).toMatchObject({
state: "ok",
reason: "configured",
params: { days: 45 },
});
});
it("does not ask a bucket that is not configured", async () => {
const { signal } = await retentionSignal();
expect(signal).toMatchObject({
state: "inactive",
reason: "not_configured",
});
expect(probe).not.toHaveBeenCalled();
});
});