diff --git a/docs/api/openapi.yaml b/docs/api/openapi.yaml index 9255e81e3..70c23f6ee 100644 --- a/docs/api/openapi.yaml +++ b/docs/api/openapi.yaml @@ -37138,6 +37138,7 @@ components: - SECURITY_ALERT - DAILY_BRIEFING - SHARED_RECORD_INTAKE + - BACKUP_HEALTH enabled: type: boolean required: @@ -38519,7 +38520,7 @@ components: documents.sources.tokenRequired, documents.sources.unavailable, documents.sources.unreachable, documents.sources.versionTooOld; encounter — encounter.invalid, encounter.link.invalid, encounter.practitioner-not-found, encounter.reminder-conflict, encounter.reminder-not-found; encryption - — encryption.keyBackup.stale, encryption.key_mismatch; environment — environment.backfill_pending, + — encryption.key_mismatch; environment — environment.backfill_pending, environment.backfill_rate_limited, environment.invalid, environment.no_home, environment.range_too_large, environment.travel.not_found, environment.travel_suggestion.not_found; export — export.selection.empty, export.selection.unknown_leaf; familyHistory — familyHistory.invalid; @@ -38580,30 +38581,30 @@ components: profile.update.emailInUse, profile.update.emailRateLimited, profile.update.emailSecondFactorRequired, profile.update.invalidBody, profile.update.nothingSaved; profile_claim — profile_claim.invalid, profile_claim.oidc_only_unsupported, profile_claim.sso_unavailable, profile_claim.taken; record_write — - record_write.rate_limited; report-selection — report-selection.body.invalid_json, - report-selection.body.invalid_shape, report-selection.leaves.unknown; route — route.retired; search — - search.invalid; security_activity — security_activity.invalid; share-link — - share-link.document.decryptFailed, share-link.document.notFound, share-link.document.rateLimited, - share-link.documents.invalid, share-link.report.notFound, share-link.report.rateLimited, - share-link.selection.forbidden_leaf, share-link.selection.unknown_leaf; sharing — - sharing.accept.expired, sharing.accept.not_found, sharing.accept.not_pending, sharing.accept.revoked, - sharing.access.denied, sharing.invite.duplicate, sharing.invite.invalid, sharing.invite.invalid_scope, - sharing.invite.manage_browser_only, sharing.invite.no_account, sharing.invite.refused, - sharing.invite.self, sharing.not_permitted, sharing.renounce.already_ended, sharing.renounce.not_found, - sharing.revoke.already_ended, sharing.revoke.not_found, sharing.session.changed, sharing.switch.invalid, - sharing.switch.wrong_transport; source-priority — source-priority.body.invalid_json, - source-priority.body.invalid_shape; symptoms — symptoms.definition.limitReached, - symptoms.definition.notFound, symptoms.episode.notOpen, symptoms.event.notFound, symptoms.invalid; - timeline — timeline.invalid; tokens — tokens.documents.ceiling_reached, - tokens.measurements.ceiling_reached, tokens.mirror.not_measurement_token, - tokens.workouts.ceiling_reached; vaccination — vaccination.booster-invalid, - vaccination.booster-no-antigen, vaccination.booster-out-of-scope, vaccination.custom-vaccine-not-found, - vaccination.custom.invalid, vaccination.custom.name-taken, vaccination.custom.not-found, - vaccination.encounter-not-found, vaccination.identity-required, vaccination.invalid, - vaccination.link.invalid, vaccination.not-found, vaccination.practitioner-not-found; workout — - workout.batch.invalid, workout.batch.payload_too_large, workout.batch.source_not_permitted, - workout.batch.too_large, workout.delete.synced_source; (unprefixed) — about_me_conflict, - ai_response_truncated, coach_prefs_conflict, dashboard_layout_conflict, empty_file, + record_write.rate_limited; recovery — recovery.kit.stale; report-selection — + report-selection.body.invalid_json, report-selection.body.invalid_shape, + report-selection.leaves.unknown; route — route.retired; search — search.invalid; security_activity — + security_activity.invalid; share-link — share-link.document.decryptFailed, share-link.document.notFound, + share-link.document.rateLimited, share-link.documents.invalid, share-link.report.notFound, + share-link.report.rateLimited, share-link.selection.forbidden_leaf, share-link.selection.unknown_leaf; + sharing — sharing.accept.expired, sharing.accept.not_found, sharing.accept.not_pending, + sharing.accept.revoked, sharing.access.denied, sharing.invite.duplicate, sharing.invite.invalid, + sharing.invite.invalid_scope, sharing.invite.manage_browser_only, sharing.invite.no_account, + sharing.invite.refused, sharing.invite.self, sharing.not_permitted, sharing.renounce.already_ended, + sharing.renounce.not_found, sharing.revoke.already_ended, sharing.revoke.not_found, + sharing.session.changed, sharing.switch.invalid, sharing.switch.wrong_transport; source-priority — + source-priority.body.invalid_json, source-priority.body.invalid_shape; symptoms — + symptoms.definition.limitReached, symptoms.definition.notFound, symptoms.episode.notOpen, + symptoms.event.notFound, symptoms.invalid; timeline — timeline.invalid; tokens — + tokens.documents.ceiling_reached, tokens.measurements.ceiling_reached, + tokens.mirror.not_measurement_token, tokens.workouts.ceiling_reached; vaccination — + vaccination.booster-invalid, vaccination.booster-no-antigen, vaccination.booster-out-of-scope, + vaccination.custom-vaccine-not-found, vaccination.custom.invalid, vaccination.custom.name-taken, + vaccination.custom.not-found, vaccination.encounter-not-found, vaccination.identity-required, + vaccination.invalid, vaccination.link.invalid, vaccination.not-found, + vaccination.practitioner-not-found; workout — workout.batch.invalid, workout.batch.payload_too_large, + workout.batch.source_not_permitted, workout.batch.too_large, workout.delete.synced_source; (unprefixed) + — about_me_conflict, ai_response_truncated, coach_prefs_conflict, dashboard_layout_conflict, empty_file, health_score_config_conflict, insights_layout_conflict, invalid_base_updated_at, json_carries_no_intake_time, json_not_an_array, medication_layout_conflict, missing_required_columns, modules_conflict, mood_tag_layout_conflict, not_configured, notification_prefs_conflict, oidc_only, @@ -87952,6 +87953,7 @@ components: - SECURITY_ALERT - DAILY_BRIEFING - SHARED_RECORD_INTAKE + - BACKUP_HEALTH description: Notification category a per-channel preference can toggle. additionalProperties: type: boolean @@ -87973,6 +87975,7 @@ components: - SECURITY_ALERT - DAILY_BRIEFING - SHARED_RECORD_INTAKE + - BACKUP_HEALTH description: What each event does on a channel with no stored row, resolved for this account by the function delivery uses. Most are constant per category; the mood reminder follows the account's mood-reminder setting (`moodReminderEnabled` on `/api/user/profile`). Render a switch without a row from this map rather than from @@ -88043,6 +88046,7 @@ components: - SECURITY_ALERT - DAILY_BRIEFING - SHARED_RECORD_INTAKE + - BACKUP_HEALTH description: Notification category a per-channel preference can toggle. enabled: type: boolean diff --git a/messages/de.json b/messages/de.json index 11ced69c4..5bbe197e2 100644 --- a/messages/de.json +++ b/messages/de.json @@ -8606,6 +8606,96 @@ "confirmedAt": "Bestätigt am {date}.", "confirmedAtBy": "Bestätigt am {date} von {who}.", "dueAgain": "Das Kit wurde am {date} bestätigt, führt jetzt aber Schlüssel auf, die diese Bestätigung nicht abdeckt: {keys}." + }, + "recoveryReadiness": { + "title": "Wiederherstellbarkeit", + "description": "Ob sich dieser Server nach einem Ausfall wiederherstellen ließe, beurteilt anhand jeder Sicherung, die er anlegt.", + "loading": "Sicherungen werden geprüft…", + "loadError": "Die Wiederherstellbarkeit konnte nicht geladen werden.", + "state": { + "ok": "Bereit", + "attention": "Braucht Aufmerksamkeit", + "failing": "Gefährdet" + }, + "signal": { + "database_dump": "Datenbank-Dump", + "weekly_copy": "Wöchentliche Kontokopien", + "offhost_copies": "Externe Kopien", + "restore_drill": "Wiederherstellungsprobe", + "offhost_setup": "Externe Sicherung", + "recovery_kit": "Wiederherstellungs-Kit" + }, + "at": { + "database_dump": "Neuester Dump: {date}", + "weekly_copy": "Älteste Kontokopie: {date}", + "offhost_copies": "Älteste externe Kopie: {date}", + "restore_drill": "Letzte Probe: {date}", + "recovery_kit": "Bestätigt: {date}" + }, + "dueBy": "Fällig bis {date}", + "fix": { + "database_dump": "Sicherungen öffnen", + "weekly_copy": "Sicherungen öffnen", + "offhost_copies": "Sicherungen öffnen", + "restore_drill": "Sicherungen öffnen", + "offhost_setup": "Sicherungen öffnen", + "recovery_kit": "Kit öffnen" + }, + "reason": { + "database_dump": { + "not_available": "Diese Version legt keinen geplanten Datenbank-Dump an.", + "disabled": "Der geplante Datenbank-Dump ist ausgeschaltet.", + "unreadable": "Der Zustand des Datenbank-Dumps konnte nicht gelesen werden.", + "never_succeeded": "Noch kein Datenbank-Dump ist gelungen, und der letzte Versuch ist fehlgeschlagen.", + "never_ran": "In den {hours} Stunden, seit Dumps eingeschaltet sind, wurde kein Datenbank-Dump geschrieben.", + "first_pending": "Der erste Datenbank-Dump ist noch nicht gelaufen.", + "stale": "Der neueste Datenbank-Dump ist {hours} Stunden alt.", + "last_failed": "Der letzte Datenbank-Dump ist fehlgeschlagen. Der davor ist noch aktuell.", + "fresh": "Der neueste Datenbank-Dump ist aktuell.", + "nowhere_to_write": "Das Dump-Verzeichnis {directory} ist nicht beschreibbar und es ist kein externer Bucket eingerichtet, daher kann kein Dump aufbewahrt werden." + }, + "weekly_copy": { + "no_accounts": "Es gibt noch keine Konten zum Kopieren.", + "stale": "Die älteste wöchentliche Kopie ist {days} Tage alt. Konten im Rückstand: {accounts}.", + "never": "Ein Konto hat seit {days} Tagen keine wöchentliche Kopie. Konten im Rückstand: {accounts}.", + "first_pending": "Die erste wöchentliche Kopie entsteht am frühen Sonntagmorgen.", + "fresh": "Jedes Konto hat eine Kopie aus der letzten Woche." + }, + "offhost_copies": { + "not_configured": "Es ist kein externer Bucket eingerichtet.", + "stale": "Externe Kopien liegen mehr als zwei Nächte zurück. Konten im Rückstand: {accounts}.", + "first_pending": "Der nächtliche Lauf für externe Kopien hat noch kein Konto erreicht.", + "fresh": "Jedes Konto hat eine aktuelle Kopie im externen Bucket." + }, + "restore_drill": { + "not_configured": "Die Probe liest den externen Bucket, und der ist nicht eingerichtet.", + "not_run_yet": "Die monatliche Wiederherstellungsprobe ist noch nicht gelaufen.", + "failed": "Die letzte Wiederherstellungsprobe ist fehlgeschlagen: {failed} von {checked} Kopien ließen sich nicht zurücklesen.", + "failed_run": "Die letzte Wiederherstellungsprobe konnte nicht zu Ende laufen.", + "passed": "Die letzte Wiederherstellungsprobe hat {checked} Kopien zurückgelesen." + }, + "offhost_setup": { + "configured": "Kopien verlassen diesen Server jede Nacht.", + "no_admin": "Es gibt noch kein Admin-Konto.", + "not_configured": "Es ist kein externer Bucket eingerichtet, daher liegt jede Kopie nur auf diesem Server.", + "not_configured_grace": "Es ist noch kein externer Bucket eingerichtet, daher liegt jede Kopie nur auf diesem Server." + }, + "recovery_kit": { + "confirmed": "Von jedem Schlüssel, den eine Wiederherstellung braucht, gibt es eine Kopie.", + "out_of_date": "Das Wiederherstellungs-Kit führt Schlüssel auf, für die niemand eine Kopie bestätigt hat: {keys}.", + "no_admin": "Es gibt noch kein Admin-Konto.", + "not_confirmed": "Niemand hat eine Kopie der Schlüssel bestätigt, die eine Wiederherstellung braucht: {keys}.", + "not_confirmed_grace": "Sichere eine Kopie der Schlüssel, die eine Wiederherstellung braucht: {keys}." + } + }, + "recovered": { + "database_dump": "Der Datenbank-Dump ist wieder aktuell.", + "weekly_copy": "Jedes Konto hat wieder eine aktuelle wöchentliche Kopie.", + "offhost_copies": "Jedes Konto hat wieder eine aktuelle externe Kopie.", + "restore_drill": "Die Wiederherstellungsprobe ist wieder bestanden.", + "offhost_setup": "Externe Kopien sind jetzt eingerichtet.", + "recovery_kit": "Das Wiederherstellungs-Kit ist wieder bestätigt." + } } }, "achievements": { @@ -9071,6 +9161,14 @@ "checkupReminders": { "title": "Vorsorge-Erinnerungen", "description": "Schalte einzelne Vorsorge-Erinnerungen ein oder aus." + }, + "eventBackupHealth": "Zustand der Sicherungen", + "eventBackupHealthDesc": "Wenn eine Sicherung ausbleibt, zurückfällt oder wieder läuft. Nur für Admins.", + "recoveryReadiness": { + "titleAttention": "Sicherungen brauchen Aufmerksamkeit", + "titleFailing": "Sicherungen sind gefährdet", + "titleReminder": "Sicherungen sind weiterhin gefährdet", + "titleRecovered": "Sicherungen laufen wieder normal" } }, "i18n": { diff --git a/messages/en.json b/messages/en.json index 24d7846e2..4267cb9f5 100644 --- a/messages/en.json +++ b/messages/en.json @@ -8606,6 +8606,96 @@ "confirmedAt": "Confirmed on {date}.", "confirmedAtBy": "Confirmed on {date} by {who}.", "dueAgain": "The kit was confirmed on {date}, but it now lists keys that confirmation does not cover: {keys}." + }, + "recoveryReadiness": { + "title": "Recovery readiness", + "description": "Whether this server could be put back after a loss, judged from every backup it makes.", + "loading": "Checking the backups…", + "loadError": "Could not load the recovery readiness.", + "state": { + "ok": "Ready", + "attention": "Needs attention", + "failing": "At risk" + }, + "signal": { + "database_dump": "Database dump", + "weekly_copy": "Weekly account copies", + "offhost_copies": "Off-host copies", + "restore_drill": "Restore drill", + "offhost_setup": "Off-host backup", + "recovery_kit": "Recovery kit" + }, + "at": { + "database_dump": "Newest dump: {date}", + "weekly_copy": "Oldest account copy: {date}", + "offhost_copies": "Least recent off-host copy: {date}", + "restore_drill": "Last drill: {date}", + "recovery_kit": "Confirmed: {date}" + }, + "dueBy": "Due by {date}", + "fix": { + "database_dump": "Open backups", + "weekly_copy": "Open backups", + "offhost_copies": "Open backups", + "restore_drill": "Open backups", + "offhost_setup": "Open backups", + "recovery_kit": "Open the kit" + }, + "reason": { + "database_dump": { + "not_available": "This version does not schedule a database dump.", + "disabled": "The scheduled database dump is switched off.", + "unreadable": "The status of the database dump could not be read.", + "never_succeeded": "No database dump has succeeded yet, and the last attempt failed.", + "never_ran": "No database dump has been written in the {hours} hours since dumps were switched on.", + "first_pending": "The first database dump has not run yet.", + "stale": "The newest database dump is {hours} hours old.", + "last_failed": "The last database dump failed. The one before it is still recent.", + "fresh": "The newest database dump is recent.", + "nowhere_to_write": "The dump directory {directory} cannot be written to and no off-host bucket is set up, so no dump can be kept." + }, + "weekly_copy": { + "no_accounts": "There are no accounts to copy yet.", + "stale": "The oldest weekly copy is {days} days old. Accounts behind: {accounts}.", + "never": "An account has had no weekly copy for {days} days. Accounts behind: {accounts}.", + "first_pending": "The first weekly copy is made early on Sunday.", + "fresh": "Every account has a copy from the last week." + }, + "offhost_copies": { + "not_configured": "No off-host bucket is set up.", + "stale": "Off-host copies are more than two nights behind. Accounts behind: {accounts}.", + "first_pending": "The nightly off-host run has not reached any account yet.", + "fresh": "Every account has a recent copy in the off-host bucket." + }, + "restore_drill": { + "not_configured": "The drill reads the off-host bucket, which is not set up.", + "not_run_yet": "The monthly restore drill has not run yet.", + "failed": "The last restore drill failed: {failed} of {checked} copies could not be read back.", + "failed_run": "The last restore drill could not finish.", + "passed": "The last restore drill read back {checked} copies." + }, + "offhost_setup": { + "configured": "Copies leave this server every night.", + "no_admin": "There is no admin account yet.", + "not_configured": "No off-host bucket is set up, so every copy lives on this server only.", + "not_configured_grace": "No off-host bucket is set up yet, so every copy lives on this server only." + }, + "recovery_kit": { + "confirmed": "A copy of every key a restore needs is kept.", + "out_of_date": "The recovery kit lists keys nobody has confirmed a copy of: {keys}.", + "no_admin": "There is no admin account yet.", + "not_confirmed": "Nobody has confirmed a copy of the keys a restore needs: {keys}.", + "not_confirmed_grace": "Save a copy of the keys a restore needs: {keys}." + } + }, + "recovered": { + "database_dump": "The database dump is recent again.", + "weekly_copy": "Every account has a recent weekly copy again.", + "offhost_copies": "Every account has a recent off-host copy again.", + "restore_drill": "The restore drill passed again.", + "offhost_setup": "Off-host copies are set up now.", + "recovery_kit": "The recovery kit is confirmed again." + } } }, "achievements": { @@ -9071,6 +9161,14 @@ "checkupReminders": { "title": "Checkup reminders", "description": "Switch individual checkup reminders on or off." + }, + "eventBackupHealth": "Backup health", + "eventBackupHealthDesc": "When a backup stops, falls behind or recovers. Sent to admins only.", + "recoveryReadiness": { + "titleAttention": "Backups need attention", + "titleFailing": "Backups are at risk", + "titleReminder": "Backups are still at risk", + "titleRecovered": "Backups are back to normal" } }, "i18n": { diff --git a/messages/es.json b/messages/es.json index 6f3591693..629461a17 100644 --- a/messages/es.json +++ b/messages/es.json @@ -8606,6 +8606,96 @@ "confirmedAt": "Confirmado el {date}.", "confirmedAtBy": "Confirmado el {date} por {who}.", "dueAgain": "El kit se confirmó el {date}, pero ahora muestra claves que esa confirmación no cubre: {keys}." + }, + "recoveryReadiness": { + "title": "Preparación para la recuperación", + "description": "Si este servidor se podría recuperar tras una pérdida, a juzgar por cada copia de seguridad que hace.", + "loading": "Comprobando las copias de seguridad…", + "loadError": "No se pudo cargar la preparación para la recuperación.", + "state": { + "ok": "Listo", + "attention": "Necesita atención", + "failing": "En riesgo" + }, + "signal": { + "database_dump": "Volcado de la base de datos", + "weekly_copy": "Copias semanales de las cuentas", + "offhost_copies": "Copias fuera del host", + "restore_drill": "Simulacro de restauración", + "offhost_setup": "Copia fuera del host", + "recovery_kit": "Kit de recuperación" + }, + "at": { + "database_dump": "Volcado más reciente: {date}", + "weekly_copy": "Copia de cuenta más antigua: {date}", + "offhost_copies": "Copia fuera del host más antigua: {date}", + "restore_drill": "Último simulacro: {date}", + "recovery_kit": "Confirmado: {date}" + }, + "dueBy": "Vence el {date}", + "fix": { + "database_dump": "Abrir copias de seguridad", + "weekly_copy": "Abrir copias de seguridad", + "offhost_copies": "Abrir copias de seguridad", + "restore_drill": "Abrir copias de seguridad", + "offhost_setup": "Abrir copias de seguridad", + "recovery_kit": "Abrir el kit" + }, + "reason": { + "database_dump": { + "not_available": "Esta versión no programa un volcado de la base de datos.", + "disabled": "El volcado programado de la base de datos está desactivado.", + "unreadable": "No se pudo leer el estado del volcado de la base de datos.", + "never_succeeded": "Ningún volcado de la base de datos ha salido bien todavía, y el último intento falló.", + "never_ran": "No se ha escrito ningún volcado de la base de datos en las {hours} horas transcurridas desde que se activaron los volcados.", + "first_pending": "El primer volcado de la base de datos aún no se ha ejecutado.", + "stale": "El volcado de la base de datos más reciente tiene {hours} horas.", + "last_failed": "El último volcado de la base de datos falló. El anterior sigue siendo reciente.", + "fresh": "El último volcado de la base de datos es reciente.", + "nowhere_to_write": "No se puede escribir en el directorio de volcados {directory} y no hay ningún bucket externo configurado, así que no se puede guardar ningún volcado." + }, + "weekly_copy": { + "no_accounts": "Todavía no hay cuentas que copiar.", + "stale": "La copia semanal más antigua tiene {days} días. Cuentas con retraso: {accounts}.", + "never": "Una cuenta lleva {days} días sin copia semanal. Cuentas con retraso: {accounts}.", + "first_pending": "La primera copia semanal se hace el domingo de madrugada.", + "fresh": "Cada cuenta tiene una copia de la última semana." + }, + "offhost_copies": { + "not_configured": "No hay ningún bucket fuera del host configurado.", + "stale": "Las copias fuera del host llevan más de dos noches de retraso. Cuentas con retraso: {accounts}.", + "first_pending": "La ejecución nocturna fuera del host aún no ha llegado a ninguna cuenta.", + "fresh": "Cada cuenta tiene una copia reciente en el bucket fuera del host." + }, + "restore_drill": { + "not_configured": "El simulacro lee el bucket fuera del host, que no está configurado.", + "not_run_yet": "El simulacro de restauración mensual aún no se ha ejecutado.", + "failed": "El último simulacro de restauración falló: no se pudieron volver a leer {failed} de {checked} copias.", + "failed_run": "El último simulacro de restauración no pudo terminar.", + "passed": "El último simulacro de restauración volvió a leer {checked} copias." + }, + "offhost_setup": { + "configured": "Las copias salen de este servidor cada noche.", + "no_admin": "Todavía no hay ninguna cuenta de administrador.", + "not_configured": "No hay ningún bucket fuera del host configurado, así que cada copia está solo en este servidor.", + "not_configured_grace": "Aún no hay ningún bucket fuera del host configurado, así que cada copia está solo en este servidor." + }, + "recovery_kit": { + "confirmed": "Se guarda una copia de cada clave que necesita una restauración.", + "out_of_date": "El kit de recuperación muestra claves de las que nadie ha confirmado una copia: {keys}.", + "no_admin": "Todavía no hay ninguna cuenta de administrador.", + "not_confirmed": "Nadie ha confirmado una copia de las claves que necesita una restauración: {keys}.", + "not_confirmed_grace": "Guarda una copia de las claves que necesita una restauración: {keys}." + } + }, + "recovered": { + "database_dump": "El volcado de la base de datos vuelve a ser reciente.", + "weekly_copy": "Cada cuenta vuelve a tener una copia semanal reciente.", + "offhost_copies": "Cada cuenta vuelve a tener una copia fuera del host reciente.", + "restore_drill": "El simulacro de restauración vuelve a superarse.", + "offhost_setup": "Las copias fuera del host ya están configuradas.", + "recovery_kit": "El kit de recuperación vuelve a estar confirmado." + } } }, "achievements": { @@ -9071,6 +9161,14 @@ "checkupReminders": { "title": "Recordatorios de chequeos", "description": "Activa o desactiva recordatorios de chequeos concretos." + }, + "eventBackupHealth": "Estado de las copias de seguridad", + "eventBackupHealthDesc": "Cuando una copia de seguridad se detiene, se retrasa o se recupera. Solo se envía a los administradores.", + "recoveryReadiness": { + "titleAttention": "Las copias de seguridad necesitan atención", + "titleFailing": "Las copias de seguridad están en riesgo", + "titleReminder": "Las copias de seguridad siguen en riesgo", + "titleRecovered": "Las copias de seguridad vuelven a la normalidad" } }, "i18n": { diff --git a/messages/fr.json b/messages/fr.json index 6fd4d7820..17c13ff4b 100644 --- a/messages/fr.json +++ b/messages/fr.json @@ -8606,6 +8606,96 @@ "confirmedAt": "Confirmé le {date}.", "confirmedAtBy": "Confirmé le {date} par {who}.", "dueAgain": "Le kit a été confirmé le {date}, mais il liste maintenant des clés que cette confirmation ne couvre pas : {keys}." + }, + "recoveryReadiness": { + "title": "Capacité de restauration", + "description": "Indique si ce serveur pourrait être remis en route après une perte, d'après chaque sauvegarde qu'il réalise.", + "loading": "Vérification des sauvegardes…", + "loadError": "Impossible de charger la capacité de restauration.", + "state": { + "ok": "Prêt", + "attention": "Demande de l'attention", + "failing": "Menacé" + }, + "signal": { + "database_dump": "Dump de la base de données", + "weekly_copy": "Copies hebdomadaires des comptes", + "offhost_copies": "Copies hors hôte", + "restore_drill": "Test de restauration", + "offhost_setup": "Sauvegarde hors hôte", + "recovery_kit": "Kit de récupération" + }, + "at": { + "database_dump": "Dump le plus récent : {date}", + "weekly_copy": "Copie de compte la plus ancienne : {date}", + "offhost_copies": "Copie hors hôte la plus ancienne : {date}", + "restore_drill": "Dernier test : {date}", + "recovery_kit": "Confirmé : {date}" + }, + "dueBy": "À faire avant le {date}", + "fix": { + "database_dump": "Ouvrir les sauvegardes", + "weekly_copy": "Ouvrir les sauvegardes", + "offhost_copies": "Ouvrir les sauvegardes", + "restore_drill": "Ouvrir les sauvegardes", + "offhost_setup": "Ouvrir les sauvegardes", + "recovery_kit": "Ouvrir le kit" + }, + "reason": { + "database_dump": { + "not_available": "Cette version ne planifie pas de dump de la base de données.", + "disabled": "Le dump planifié de la base de données est désactivé.", + "unreadable": "L'état du dump de la base de données n'a pas pu être lu.", + "never_succeeded": "Aucun dump de la base de données n'a encore réussi, et la dernière tentative a échoué.", + "never_ran": "Aucun dump de la base de données n'a été écrit dans les {hours} heures écoulées depuis l'activation des dumps.", + "first_pending": "Le premier dump de la base de données n'a pas encore été lancé.", + "stale": "Le dump de la base de données le plus récent date de {hours} heures.", + "last_failed": "Le dernier dump de la base de données a échoué. Le précédent est encore récent.", + "fresh": "Le dernier dump de la base de données est récent.", + "nowhere_to_write": "Le dossier des dumps {directory} n'est pas accessible en écriture et aucun bucket externe n'est configuré, donc aucun dump ne peut être conservé." + }, + "weekly_copy": { + "no_accounts": "Il n'y a pas encore de compte à copier.", + "stale": "La copie hebdomadaire la plus ancienne date de {days} jours. Comptes en retard : {accounts}.", + "never": "Un compte n'a pas eu de copie hebdomadaire depuis {days} jours. Comptes en retard : {accounts}.", + "first_pending": "La première copie hebdomadaire est faite tôt le dimanche matin.", + "fresh": "Chaque compte a une copie de la semaine dernière." + }, + "offhost_copies": { + "not_configured": "Aucun bucket hors hôte n'est configuré.", + "stale": "Les copies hors hôte ont plus de deux nuits de retard. Comptes en retard : {accounts}.", + "first_pending": "L'envoi nocturne hors hôte n'a encore atteint aucun compte.", + "fresh": "Chaque compte a une copie récente dans le bucket hors hôte." + }, + "restore_drill": { + "not_configured": "Le test lit le bucket hors hôte, qui n'est pas configuré.", + "not_run_yet": "Le test de restauration mensuel n'a pas encore été lancé.", + "failed": "Le dernier test de restauration a échoué : {failed} copies sur {checked} n'ont pas pu être relues.", + "failed_run": "Le dernier test de restauration n'a pas pu aller jusqu'au bout.", + "passed": "Le dernier test de restauration a relu {checked} copies." + }, + "offhost_setup": { + "configured": "Les copies quittent ce serveur chaque nuit.", + "no_admin": "Il n'y a pas encore de compte administrateur.", + "not_configured": "Aucun bucket hors hôte n'est configuré, donc chaque copie se trouve uniquement sur ce serveur.", + "not_configured_grace": "Aucun bucket hors hôte n'est encore configuré, donc chaque copie se trouve uniquement sur ce serveur." + }, + "recovery_kit": { + "confirmed": "Une copie de chaque clé nécessaire à une restauration est conservée.", + "out_of_date": "Le kit de récupération liste des clés dont personne n'a confirmé de copie : {keys}.", + "no_admin": "Il n'y a pas encore de compte administrateur.", + "not_confirmed": "Personne n'a confirmé de copie des clés nécessaires à une restauration : {keys}.", + "not_confirmed_grace": "Sauvegarde une copie des clés nécessaires à une restauration : {keys}." + } + }, + "recovered": { + "database_dump": "Le dump de la base de données est de nouveau récent.", + "weekly_copy": "Chaque compte a de nouveau une copie hebdomadaire récente.", + "offhost_copies": "Chaque compte a de nouveau une copie hors hôte récente.", + "restore_drill": "Le test de restauration réussit de nouveau.", + "offhost_setup": "Les copies hors hôte sont maintenant configurées.", + "recovery_kit": "Le kit de récupération est de nouveau confirmé." + } } }, "achievements": { @@ -9071,6 +9161,14 @@ "checkupReminders": { "title": "Rappels de bilans", "description": "Activez ou désactivez chaque rappel de bilan." + }, + "eventBackupHealth": "État des sauvegardes", + "eventBackupHealthDesc": "Quand une sauvegarde s'arrête, prend du retard ou repart. Envoyé uniquement aux administrateurs.", + "recoveryReadiness": { + "titleAttention": "Les sauvegardes demandent de l'attention", + "titleFailing": "Les sauvegardes sont menacées", + "titleReminder": "Les sauvegardes sont toujours menacées", + "titleRecovered": "Les sauvegardes sont revenues à la normale" } }, "i18n": { diff --git a/messages/it.json b/messages/it.json index ee227f3a3..e8faea848 100644 --- a/messages/it.json +++ b/messages/it.json @@ -8606,6 +8606,96 @@ "confirmedAt": "Confermato il {date}.", "confirmedAtBy": "Confermato il {date} da {who}.", "dueAgain": "Il kit è stato confermato il {date}, ma ora elenca chiavi che quella conferma non copre: {keys}." + }, + "recoveryReadiness": { + "title": "Preparazione al ripristino", + "description": "Se questo server potrebbe essere rimesso in piedi dopo una perdita, valutato in base a ogni backup che esegue.", + "loading": "Controllo dei backup…", + "loadError": "Impossibile caricare la preparazione al ripristino.", + "state": { + "ok": "Pronto", + "attention": "Richiede attenzione", + "failing": "A rischio" + }, + "signal": { + "database_dump": "Dump del database", + "weekly_copy": "Copie settimanali degli account", + "offhost_copies": "Copie fuori host", + "restore_drill": "Prova di ripristino", + "offhost_setup": "Backup fuori host", + "recovery_kit": "Kit di recupero" + }, + "at": { + "database_dump": "Dump più recente: {date}", + "weekly_copy": "Copia di account più vecchia: {date}", + "offhost_copies": "Copia fuori host meno recente: {date}", + "restore_drill": "Ultima prova: {date}", + "recovery_kit": "Confermato: {date}" + }, + "dueBy": "Scadenza: {date}", + "fix": { + "database_dump": "Apri i backup", + "weekly_copy": "Apri i backup", + "offhost_copies": "Apri i backup", + "restore_drill": "Apri i backup", + "offhost_setup": "Apri i backup", + "recovery_kit": "Apri il kit" + }, + "reason": { + "database_dump": { + "not_available": "Questa versione non pianifica un dump del database.", + "disabled": "Il dump pianificato del database è disattivato.", + "unreadable": "Non è stato possibile leggere lo stato del dump del database.", + "never_succeeded": "Nessun dump del database è ancora riuscito, e l'ultimo tentativo non è andato a buon fine.", + "never_ran": "Nelle {hours} ore trascorse da quando i dump sono stati attivati non è stato scritto alcun dump del database.", + "first_pending": "Il primo dump del database non è ancora stato eseguito.", + "stale": "Il dump del database più recente ha {hours} ore.", + "last_failed": "L'ultimo dump del database non è riuscito. Quello precedente è ancora recente.", + "fresh": "L'ultimo dump del database è recente.", + "nowhere_to_write": "La cartella dei dump {directory} non è scrivibile e non è configurato nessun bucket esterno, quindi non si può conservare alcun dump." + }, + "weekly_copy": { + "no_accounts": "Non ci sono ancora account da copiare.", + "stale": "La copia settimanale più vecchia ha {days} giorni. Account in ritardo: {accounts}.", + "never": "Un account non ha una copia settimanale da {days} giorni. Account in ritardo: {accounts}.", + "first_pending": "La prima copia settimanale viene creata domenica mattina presto.", + "fresh": "Ogni account ha una copia dell'ultima settimana." + }, + "offhost_copies": { + "not_configured": "Non è configurato alcun bucket fuori host.", + "stale": "Le copie fuori host sono indietro di più di due notti. Account in ritardo: {accounts}.", + "first_pending": "Il giro notturno fuori host non ha ancora raggiunto alcun account.", + "fresh": "Ogni account ha una copia recente nel bucket fuori host." + }, + "restore_drill": { + "not_configured": "La prova legge il bucket fuori host, che non è configurato.", + "not_run_yet": "La prova di ripristino mensile non è ancora stata eseguita.", + "failed": "L'ultima prova di ripristino non è riuscita: {failed} copie su {checked} non si sono potute rileggere.", + "failed_run": "L'ultima prova di ripristino non è riuscita ad arrivare in fondo.", + "passed": "L'ultima prova di ripristino ha riletto {checked} copie." + }, + "offhost_setup": { + "configured": "Le copie lasciano questo server ogni notte.", + "no_admin": "Non esiste ancora un account amministratore.", + "not_configured": "Non è configurato alcun bucket fuori host, quindi ogni copia si trova solo su questo server.", + "not_configured_grace": "Non è ancora configurato alcun bucket fuori host, quindi ogni copia si trova solo su questo server." + }, + "recovery_kit": { + "confirmed": "Esiste una copia di ogni chiave necessaria per un ripristino.", + "out_of_date": "Il kit di recupero elenca chiavi di cui nessuno ha confermato una copia: {keys}.", + "no_admin": "Non esiste ancora un account amministratore.", + "not_confirmed": "Nessuno ha confermato una copia delle chiavi necessarie per un ripristino: {keys}.", + "not_confirmed_grace": "Salva una copia delle chiavi necessarie per un ripristino: {keys}." + } + }, + "recovered": { + "database_dump": "Il dump del database è di nuovo recente.", + "weekly_copy": "Ogni account ha di nuovo una copia settimanale recente.", + "offhost_copies": "Ogni account ha di nuovo una copia fuori host recente.", + "restore_drill": "La prova di ripristino è di nuovo riuscita.", + "offhost_setup": "Le copie fuori host ora sono configurate.", + "recovery_kit": "Il kit di recupero è di nuovo confermato." + } } }, "achievements": { @@ -9071,6 +9161,14 @@ "checkupReminders": { "title": "Promemoria dei controlli", "description": "Attiva o disattiva i singoli promemoria dei controlli." + }, + "eventBackupHealth": "Stato dei backup", + "eventBackupHealthDesc": "Quando un backup si ferma, resta indietro o torna a funzionare. Inviato solo agli amministratori.", + "recoveryReadiness": { + "titleAttention": "I backup richiedono attenzione", + "titleFailing": "I backup sono a rischio", + "titleReminder": "I backup sono ancora a rischio", + "titleRecovered": "I backup sono tornati alla normalità" } }, "i18n": { diff --git a/messages/ko.json b/messages/ko.json index 26eafeeef..d70aa8aa4 100644 --- a/messages/ko.json +++ b/messages/ko.json @@ -8606,6 +8606,96 @@ "confirmedAt": "{date}에 확인됨.", "confirmedAtBy": "{date}에 {who}님이 확인함.", "dueAgain": "키트는 {date}에 확인되었지만 지금은 그 확인에 포함되지 않은 키가 있습니다: {keys}." + }, + "recoveryReadiness": { + "title": "복구 준비 상태", + "description": "이 서버가 만드는 모든 백업을 기준으로, 서버를 잃었을 때 되살릴 수 있는지 판단합니다.", + "loading": "백업을 확인하는 중…", + "loadError": "복구 준비 상태를 불러오지 못했습니다.", + "state": { + "ok": "준비됨", + "attention": "확인 필요", + "failing": "위험" + }, + "signal": { + "database_dump": "데이터베이스 덤프", + "weekly_copy": "주간 계정 사본", + "offhost_copies": "외부 보관 사본", + "restore_drill": "복원 훈련", + "offhost_setup": "외부 보관 백업", + "recovery_kit": "복구 키트" + }, + "at": { + "database_dump": "최신 덤프: {date}", + "weekly_copy": "가장 오래된 계정 사본: {date}", + "offhost_copies": "가장 오래된 외부 보관 사본: {date}", + "restore_drill": "마지막 훈련: {date}", + "recovery_kit": "확인: {date}" + }, + "dueBy": "{date}까지", + "fix": { + "database_dump": "백업 열기", + "weekly_copy": "백업 열기", + "offhost_copies": "백업 열기", + "restore_drill": "백업 열기", + "offhost_setup": "백업 열기", + "recovery_kit": "키트 열기" + }, + "reason": { + "database_dump": { + "not_available": "이 버전은 데이터베이스 덤프를 예약하지 않습니다.", + "disabled": "예약된 데이터베이스 덤프가 꺼져 있습니다.", + "unreadable": "데이터베이스 덤프 상태를 읽지 못했습니다.", + "never_succeeded": "아직 성공한 데이터베이스 덤프가 없으며 마지막 시도도 실패했습니다.", + "never_ran": "덤프를 켠 뒤 {hours}시간 동안 데이터베이스 덤프가 기록되지 않았습니다.", + "first_pending": "첫 데이터베이스 덤프가 아직 실행되지 않았습니다.", + "stale": "최신 데이터베이스 덤프가 {hours}시간 전 것입니다.", + "last_failed": "마지막 데이터베이스 덤프가 실패했습니다. 그 전 덤프는 아직 최근 것입니다.", + "fresh": "최신 데이터베이스 덤프가 최근 것입니다.", + "nowhere_to_write": "덤프 디렉터리 {directory}에 쓸 수 없고 외부 버킷도 설정되어 있지 않아 덤프를 보관할 수 없습니다." + }, + "weekly_copy": { + "no_accounts": "아직 복사할 계정이 없습니다.", + "stale": "가장 오래된 주간 사본이 {days}일 전 것입니다. 뒤처진 계정: {accounts}.", + "never": "{days}일 동안 주간 사본이 없는 계정이 있습니다. 뒤처진 계정: {accounts}.", + "first_pending": "첫 주간 사본은 일요일 이른 시간에 만들어집니다.", + "fresh": "모든 계정에 지난주 사본이 있습니다." + }, + "offhost_copies": { + "not_configured": "외부 보관 버킷이 설정되지 않았습니다.", + "stale": "외부 보관 사본이 이틀 밤 넘게 뒤처져 있습니다. 뒤처진 계정: {accounts}.", + "first_pending": "야간 외부 보관 실행이 아직 어떤 계정에도 도달하지 않았습니다.", + "fresh": "모든 계정의 최근 사본이 외부 보관 버킷에 있습니다." + }, + "restore_drill": { + "not_configured": "훈련은 외부 보관 버킷을 읽는데, 버킷이 설정되지 않았습니다.", + "not_run_yet": "월간 복원 훈련이 아직 실행되지 않았습니다.", + "failed": "마지막 복원 훈련이 실패했습니다: 사본 {checked}개 중 {failed}개를 다시 읽지 못했습니다.", + "failed_run": "마지막 복원 훈련을 끝까지 마치지 못했습니다.", + "passed": "마지막 복원 훈련에서 사본 {checked}개를 다시 읽었습니다." + }, + "offhost_setup": { + "configured": "사본이 매일 밤 이 서버 밖으로 나갑니다.", + "no_admin": "아직 관리자 계정이 없습니다.", + "not_configured": "외부 보관 버킷이 설정되지 않아 모든 사본이 이 서버에만 있습니다.", + "not_configured_grace": "외부 보관 버킷이 아직 설정되지 않아 모든 사본이 이 서버에만 있습니다." + }, + "recovery_kit": { + "confirmed": "복원에 필요한 모든 키의 사본이 보관되어 있습니다.", + "out_of_date": "복구 키트에 아무도 사본을 확인하지 않은 키가 있습니다: {keys}.", + "no_admin": "아직 관리자 계정이 없습니다.", + "not_confirmed": "복원에 필요한 키의 사본을 아무도 확인하지 않았습니다: {keys}.", + "not_confirmed_grace": "복원에 필요한 키의 사본을 보관하세요: {keys}." + } + }, + "recovered": { + "database_dump": "데이터베이스 덤프가 다시 최신 상태입니다.", + "weekly_copy": "모든 계정에 다시 최근 주간 사본이 있습니다.", + "offhost_copies": "모든 계정에 다시 최근 외부 보관 사본이 있습니다.", + "restore_drill": "복원 훈련을 다시 통과했습니다.", + "offhost_setup": "이제 외부 보관 사본이 설정되었습니다.", + "recovery_kit": "복구 키트가 다시 확인되었습니다." + } } }, "achievements": { @@ -9071,6 +9161,14 @@ "checkupReminders": { "title": "검진 리마인더", "description": "검진 리마인더를 개별로 켜거나 끕니다." + }, + "eventBackupHealth": "백업 상태", + "eventBackupHealthDesc": "백업이 멈추거나 뒤처지거나 다시 정상으로 돌아오면 보내요. 관리자에게만 보내요.", + "recoveryReadiness": { + "titleAttention": "백업에 확인이 필요해요", + "titleFailing": "백업이 위험해요", + "titleReminder": "백업이 여전히 위험해요", + "titleRecovered": "백업이 다시 정상이에요" } }, "i18n": { diff --git a/messages/nl.json b/messages/nl.json index 1bf30323a..a05952c4b 100644 --- a/messages/nl.json +++ b/messages/nl.json @@ -8606,6 +8606,96 @@ "confirmedAt": "Bevestigd op {date}.", "confirmedAtBy": "Bevestigd op {date} door {who}.", "dueAgain": "De kit is bevestigd op {date}, maar bevat nu sleutels die niet onder die bevestiging vallen: {keys}." + }, + "recoveryReadiness": { + "title": "Herstelbaarheid", + "description": "Of deze server na verlies weer op te bouwen is, beoordeeld op basis van elke back-up die hij maakt.", + "loading": "Back-ups controleren…", + "loadError": "De herstelbaarheid kon niet worden geladen.", + "state": { + "ok": "Gereed", + "attention": "Aandacht nodig", + "failing": "In gevaar" + }, + "signal": { + "database_dump": "Databasedump", + "weekly_copy": "Wekelijkse accountkopieën", + "offhost_copies": "Externe kopieën", + "restore_drill": "Hersteltest", + "offhost_setup": "Externe back-up", + "recovery_kit": "Herstelkit" + }, + "at": { + "database_dump": "Nieuwste dump: {date}", + "weekly_copy": "Oudste accountkopie: {date}", + "offhost_copies": "Oudste externe kopie: {date}", + "restore_drill": "Laatste test: {date}", + "recovery_kit": "Bevestigd: {date}" + }, + "dueBy": "Uiterlijk {date}", + "fix": { + "database_dump": "Back-ups openen", + "weekly_copy": "Back-ups openen", + "offhost_copies": "Back-ups openen", + "restore_drill": "Back-ups openen", + "offhost_setup": "Back-ups openen", + "recovery_kit": "Kit openen" + }, + "reason": { + "database_dump": { + "not_available": "Deze versie plant geen databasedump in.", + "disabled": "De geplande databasedump is uitgeschakeld.", + "unreadable": "De status van de databasedump kon niet worden gelezen.", + "never_succeeded": "Er is nog geen databasedump gelukt, en de laatste poging is mislukt.", + "never_ran": "In de {hours} uur sinds dumps zijn ingeschakeld, is er geen databasedump geschreven.", + "first_pending": "De eerste databasedump is nog niet uitgevoerd.", + "stale": "De nieuwste databasedump is {hours} uur oud.", + "last_failed": "De laatste databasedump is mislukt. De dump daarvoor is nog recent.", + "fresh": "De nieuwste databasedump is recent.", + "nowhere_to_write": "De dumpmap {directory} is niet beschrijfbaar en er is geen externe bucket ingesteld, dus er kan geen dump worden bewaard." + }, + "weekly_copy": { + "no_accounts": "Er zijn nog geen accounts om te kopiëren.", + "stale": "De oudste wekelijkse kopie is {days} dagen oud. Achterlopende accounts: {accounts}.", + "never": "Een account heeft al {days} dagen geen wekelijkse kopie. Achterlopende accounts: {accounts}.", + "first_pending": "De eerste wekelijkse kopie wordt vroeg op zondag gemaakt.", + "fresh": "Elk account heeft een kopie van de afgelopen week." + }, + "offhost_copies": { + "not_configured": "Er is geen externe bucket ingesteld.", + "stale": "Externe kopieën lopen meer dan twee nachten achter. Achterlopende accounts: {accounts}.", + "first_pending": "De nachtelijke run voor externe kopieën heeft nog geen enkel account bereikt.", + "fresh": "Elk account heeft een recente kopie in de externe bucket." + }, + "restore_drill": { + "not_configured": "De test leest de externe bucket, en die is niet ingesteld.", + "not_run_yet": "De maandelijkse hersteltest is nog niet uitgevoerd.", + "failed": "De laatste hersteltest is mislukt: {failed} van {checked} kopieën konden niet worden teruggelezen.", + "failed_run": "De laatste hersteltest kon niet worden afgerond.", + "passed": "De laatste hersteltest heeft {checked} kopieën teruggelezen." + }, + "offhost_setup": { + "configured": "Kopieën verlaten deze server elke nacht.", + "no_admin": "Er is nog geen beheerdersaccount.", + "not_configured": "Er is geen externe bucket ingesteld, dus elke kopie staat alleen op deze server.", + "not_configured_grace": "Er is nog geen externe bucket ingesteld, dus elke kopie staat alleen op deze server." + }, + "recovery_kit": { + "confirmed": "Van elke sleutel die een herstel nodig heeft, wordt een kopie bewaard.", + "out_of_date": "De herstelkit bevat sleutels waarvan niemand een kopie heeft bevestigd: {keys}.", + "no_admin": "Er is nog geen beheerdersaccount.", + "not_confirmed": "Niemand heeft een kopie bevestigd van de sleutels die een herstel nodig heeft: {keys}.", + "not_confirmed_grace": "Bewaar een kopie van de sleutels die een herstel nodig heeft: {keys}." + } + }, + "recovered": { + "database_dump": "De databasedump is weer recent.", + "weekly_copy": "Elk account heeft weer een recente wekelijkse kopie.", + "offhost_copies": "Elk account heeft weer een recente externe kopie.", + "restore_drill": "De hersteltest is weer geslaagd.", + "offhost_setup": "Externe kopieën zijn nu ingesteld.", + "recovery_kit": "De herstelkit is weer bevestigd." + } } }, "achievements": { @@ -9071,6 +9161,14 @@ "checkupReminders": { "title": "Herinneringen voor controles", "description": "Zet afzonderlijke herinneringen voor controles aan of uit." + }, + "eventBackupHealth": "Status van back-ups", + "eventBackupHealthDesc": "Wordt verstuurd als een back-up stopt, achterloopt of weer bijgewerkt is. Alleen voor beheerders.", + "recoveryReadiness": { + "titleAttention": "Back-ups hebben aandacht nodig", + "titleFailing": "Back-ups lopen gevaar", + "titleReminder": "Back-ups lopen nog steeds gevaar", + "titleRecovered": "Back-ups zijn weer in orde" } }, "i18n": { diff --git a/messages/pl.json b/messages/pl.json index 9f07e3f4f..26d5d9e58 100644 --- a/messages/pl.json +++ b/messages/pl.json @@ -8606,6 +8606,96 @@ "confirmedAt": "Potwierdzono {date}.", "confirmedAtBy": "Potwierdzono {date}, potwierdził(a): {who}.", "dueAgain": "Zestaw potwierdzono {date}, ale teraz wymienia klucze, których to potwierdzenie nie obejmuje: {keys}." + }, + "recoveryReadiness": { + "title": "Gotowość do odzyskiwania", + "description": "Czy ten serwer dałoby się odtworzyć po utracie, oceniane na podstawie każdej kopii zapasowej, którą tworzy.", + "loading": "Sprawdzanie kopii zapasowych…", + "loadError": "Nie udało się wczytać gotowości do odzyskiwania.", + "state": { + "ok": "Gotowe", + "attention": "Wymaga uwagi", + "failing": "Zagrożone" + }, + "signal": { + "database_dump": "Zrzut bazy danych", + "weekly_copy": "Cotygodniowe kopie kont", + "offhost_copies": "Kopie poza hostem", + "restore_drill": "Próba przywracania", + "offhost_setup": "Kopia poza hostem", + "recovery_kit": "Zestaw odzyskiwania" + }, + "at": { + "database_dump": "Najnowszy zrzut: {date}", + "weekly_copy": "Najstarsza kopia konta: {date}", + "offhost_copies": "Najstarsza kopia poza hostem: {date}", + "restore_drill": "Ostatnia próba: {date}", + "recovery_kit": "Potwierdzono: {date}" + }, + "dueBy": "Termin: {date}", + "fix": { + "database_dump": "Otwórz kopie zapasowe", + "weekly_copy": "Otwórz kopie zapasowe", + "offhost_copies": "Otwórz kopie zapasowe", + "restore_drill": "Otwórz kopie zapasowe", + "offhost_setup": "Otwórz kopie zapasowe", + "recovery_kit": "Otwórz zestaw" + }, + "reason": { + "database_dump": { + "not_available": "Ta wersja nie planuje zrzutu bazy danych.", + "disabled": "Zaplanowany zrzut bazy danych jest wyłączony.", + "unreadable": "Nie udało się odczytać stanu zrzutu bazy danych.", + "never_succeeded": "Żaden zrzut bazy danych jeszcze się nie udał, a ostatnia próba zakończyła się błędem.", + "never_ran": "Od włączenia zrzutów minęło {hours} h, a nie zapisano żadnego zrzutu bazy danych.", + "first_pending": "Pierwszy zrzut bazy danych jeszcze się nie odbył.", + "stale": "Najnowszy zrzut bazy danych ma {hours} h.", + "last_failed": "Ostatni zrzut bazy danych nie powiódł się. Poprzedni jest jeszcze aktualny.", + "fresh": "Najnowszy zrzut bazy danych jest aktualny.", + "nowhere_to_write": "Katalog zrzutów {directory} nie jest zapisywalny i nie skonfigurowano zewnętrznego bucketu, więc żaden zrzut nie może zostać zachowany." + }, + "weekly_copy": { + "no_accounts": "Nie ma jeszcze kont do skopiowania.", + "stale": "Najstarsza cotygodniowa kopia ma {days} dni. Zaległe konta: {accounts}.", + "never": "Jedno z kont od {days} dni nie ma cotygodniowej kopii. Zaległe konta: {accounts}.", + "first_pending": "Pierwsza cotygodniowa kopia powstaje wcześnie rano w niedzielę.", + "fresh": "Każde konto ma kopię z ostatniego tygodnia." + }, + "offhost_copies": { + "not_configured": "Nie skonfigurowano bucketu poza hostem.", + "stale": "Kopie poza hostem są opóźnione o ponad dwie noce. Zaległe konta: {accounts}.", + "first_pending": "Nocny przebieg kopii poza hostem nie objął jeszcze żadnego konta.", + "fresh": "Każde konto ma aktualną kopię w buckecie poza hostem." + }, + "restore_drill": { + "not_configured": "Próba odczytuje bucket poza hostem, który nie jest skonfigurowany.", + "not_run_yet": "Comiesięczna próba przywracania jeszcze się nie odbyła.", + "failed": "Ostatnia próba przywracania nie powiodła się: nie udało się odczytać {failed} z {checked} kopii.", + "failed_run": "Ostatnia próba przywracania nie dobiegła końca.", + "passed": "Ostatnia próba przywracania odczytała kopie: {checked}." + }, + "offhost_setup": { + "configured": "Kopie opuszczają ten serwer każdej nocy.", + "no_admin": "Nie ma jeszcze konta administratora.", + "not_configured": "Nie skonfigurowano bucketu poza hostem, więc każda kopia jest tylko na tym serwerze.", + "not_configured_grace": "Nie skonfigurowano jeszcze bucketu poza hostem, więc każda kopia jest tylko na tym serwerze." + }, + "recovery_kit": { + "confirmed": "Istnieje kopia każdego klucza potrzebnego do przywrócenia.", + "out_of_date": "Zestaw odzyskiwania wymienia klucze, których kopii nikt nie potwierdził: {keys}.", + "no_admin": "Nie ma jeszcze konta administratora.", + "not_confirmed": "Nikt nie potwierdził kopii kluczy potrzebnych do przywrócenia: {keys}.", + "not_confirmed_grace": "Zapisz kopię kluczy potrzebnych do przywrócenia: {keys}." + } + }, + "recovered": { + "database_dump": "Zrzut bazy danych znów jest aktualny.", + "weekly_copy": "Każde konto znów ma aktualną cotygodniową kopię.", + "offhost_copies": "Każde konto znów ma aktualną kopię poza hostem.", + "restore_drill": "Próba przywracania znów się powiodła.", + "offhost_setup": "Kopie poza hostem są już skonfigurowane.", + "recovery_kit": "Zestaw odzyskiwania jest znów potwierdzony." + } } }, "achievements": { @@ -9071,6 +9161,14 @@ "checkupReminders": { "title": "Przypomnienia o badaniach kontrolnych", "description": "Włączaj lub wyłączaj pojedyncze przypomnienia o badaniach." + }, + "eventBackupHealth": "Stan kopii zapasowych", + "eventBackupHealthDesc": "Gdy kopia zapasowa przestaje powstawać, zostaje w tyle lub wraca do normy. Wysyłane tylko do administratorów.", + "recoveryReadiness": { + "titleAttention": "Kopie zapasowe wymagają uwagi", + "titleFailing": "Kopie zapasowe są zagrożone", + "titleReminder": "Kopie zapasowe są nadal zagrożone", + "titleRecovered": "Kopie zapasowe znów działają normalnie" } }, "i18n": { diff --git a/messages/pt-BR.json b/messages/pt-BR.json index 30697ec52..0ade6a831 100644 --- a/messages/pt-BR.json +++ b/messages/pt-BR.json @@ -8606,6 +8606,96 @@ "confirmedAt": "Confirmado em {date}.", "confirmedAtBy": "Confirmado em {date} por {who}.", "dueAgain": "O kit foi confirmado em {date}, mas agora lista chaves que essa confirmação não cobre: {keys}." + }, + "recoveryReadiness": { + "title": "Prontidão para recuperação", + "description": "Se este servidor poderia ser reconstruído após uma perda, avaliado a partir de cada backup que ele faz.", + "loading": "Verificando os backups…", + "loadError": "Não foi possível carregar a prontidão para recuperação.", + "state": { + "ok": "Pronto", + "attention": "Precisa de atenção", + "failing": "Em risco" + }, + "signal": { + "database_dump": "Dump do banco de dados", + "weekly_copy": "Cópias semanais das contas", + "offhost_copies": "Cópias fora do host", + "restore_drill": "Teste de restauração", + "offhost_setup": "Backup fora do host", + "recovery_kit": "Kit de recuperação" + }, + "at": { + "database_dump": "Dump mais recente: {date}", + "weekly_copy": "Cópia de conta mais antiga: {date}", + "offhost_copies": "Cópia fora do host mais antiga: {date}", + "restore_drill": "Último teste: {date}", + "recovery_kit": "Confirmado: {date}" + }, + "dueBy": "Prazo: {date}", + "fix": { + "database_dump": "Abrir backups", + "weekly_copy": "Abrir backups", + "offhost_copies": "Abrir backups", + "restore_drill": "Abrir backups", + "offhost_setup": "Abrir backups", + "recovery_kit": "Abrir o kit" + }, + "reason": { + "database_dump": { + "not_available": "Esta versão não agenda um dump do banco de dados.", + "disabled": "O dump agendado do banco de dados está desativado.", + "unreadable": "Não foi possível ler o status do dump do banco de dados.", + "never_succeeded": "Nenhum dump do banco de dados foi concluído com sucesso ainda, e a última tentativa falhou.", + "never_ran": "Nenhum dump do banco de dados foi gravado nas {hours} horas desde que os dumps foram ativados.", + "first_pending": "O primeiro dump do banco de dados ainda não foi executado.", + "stale": "O dump do banco de dados mais recente tem {hours} horas.", + "last_failed": "O último dump do banco de dados falhou. O anterior ainda é recente.", + "fresh": "O dump do banco de dados mais recente está em dia.", + "nowhere_to_write": "Não é possível gravar no diretório de dumps {directory} e nenhum bucket externo está configurado, então nenhum dump pode ser guardado." + }, + "weekly_copy": { + "no_accounts": "Ainda não há contas para copiar.", + "stale": "A cópia semanal mais antiga tem {days} dias. Contas atrasadas: {accounts}.", + "never": "Uma conta está sem cópia semanal há {days} dias. Contas atrasadas: {accounts}.", + "first_pending": "A primeira cópia semanal é feita na madrugada de domingo.", + "fresh": "Todas as contas têm uma cópia da última semana." + }, + "offhost_copies": { + "not_configured": "Nenhum bucket fora do host está configurado.", + "stale": "As cópias fora do host estão mais de duas noites atrasadas. Contas atrasadas: {accounts}.", + "first_pending": "A execução noturna fora do host ainda não chegou a nenhuma conta.", + "fresh": "Todas as contas têm uma cópia recente no bucket fora do host." + }, + "restore_drill": { + "not_configured": "O teste lê o bucket fora do host, que não está configurado.", + "not_run_yet": "O teste mensal de restauração ainda não foi executado.", + "failed": "O último teste de restauração falhou: {failed} de {checked} cópias não puderam ser lidas de volta.", + "failed_run": "O último teste de restauração não pôde ser concluído.", + "passed": "O último teste de restauração leu de volta {checked} cópias." + }, + "offhost_setup": { + "configured": "As cópias saem deste servidor todas as noites.", + "no_admin": "Ainda não há uma conta de administrador.", + "not_configured": "Nenhum bucket fora do host está configurado, então todas as cópias ficam apenas neste servidor.", + "not_configured_grace": "Nenhum bucket fora do host foi configurado ainda, então todas as cópias ficam apenas neste servidor." + }, + "recovery_kit": { + "confirmed": "Há uma cópia guardada de cada chave de que uma restauração precisa.", + "out_of_date": "O kit de recuperação lista chaves das quais ninguém confirmou uma cópia: {keys}.", + "no_admin": "Ainda não há uma conta de administrador.", + "not_confirmed": "Ninguém confirmou uma cópia das chaves de que uma restauração precisa: {keys}.", + "not_confirmed_grace": "Guarde uma cópia das chaves de que uma restauração precisa: {keys}." + } + }, + "recovered": { + "database_dump": "O dump do banco de dados está em dia novamente.", + "weekly_copy": "Todas as contas têm novamente uma cópia semanal recente.", + "offhost_copies": "Todas as contas têm novamente uma cópia recente fora do host.", + "restore_drill": "O teste de restauração passou novamente.", + "offhost_setup": "As cópias fora do host estão configuradas agora.", + "recovery_kit": "O kit de recuperação está confirmado novamente." + } } }, "achievements": { @@ -9071,6 +9161,14 @@ "checkupReminders": { "title": "Lembretes de check-up", "description": "Ligue ou desligue lembretes de check-up individuais." + }, + "eventBackupHealth": "Saúde dos backups", + "eventBackupHealthDesc": "Enviado quando um backup para, fica atrasado ou se recupera. Apenas para administradores.", + "recoveryReadiness": { + "titleAttention": "Os backups precisam de atenção", + "titleFailing": "Os backups estão em risco", + "titleReminder": "Os backups continuam em risco", + "titleRecovered": "Os backups voltaram ao normal" } }, "i18n": { diff --git a/src/__tests__/success-affordance-guard.test.ts b/src/__tests__/success-affordance-guard.test.ts index 78eea0bbb..f271e4b2e 100644 --- a/src/__tests__/success-affordance-guard.test.ts +++ b/src/__tests__/success-affordance-guard.test.ts @@ -135,6 +135,13 @@ const PINNED_AFFORDANCES: Record< // off-host copy is, and the state it paints green came out of the database // rather than out of a request this card just made. "src/components/admin/offhost-backups-section.tsx": { "text-success": 1 }, + // Reads the server's verdict, writes nothing: the check and the green mark + // say a backup signal is ok as the readiness model judged it, not that a + // request this card made succeeded. + "src/components/admin/recovery-readiness-card.tsx": { + CheckCircle2: 2, + "text-success": 1, + }, "src/components/admin/recent-audit-preview.tsx": { CheckCircle2: 2, "text-success": 1, diff --git a/src/app/admin/[section]/renderer.tsx b/src/app/admin/[section]/renderer.tsx index fc35e8d16..97f3aaba9 100644 --- a/src/app/admin/[section]/renderer.tsx +++ b/src/app/admin/[section]/renderer.tsx @@ -28,6 +28,7 @@ import { ApiTokenOverviewSection } from "@/components/admin/api-token-overview-s import { AppLogPreviewSection } from "@/components/admin/app-log-preview-section"; import { BackupsSection } from "@/components/admin/backups-section"; import { OffhostBackupsSection } from "@/components/admin/offhost-backups-section"; +import { RecoveryReadinessCard } from "@/components/admin/recovery-readiness-card"; import { DangerZoneSection } from "@/components/admin/danger-zone-section"; import { EncryptionSection } from "@/components/admin/encryption-section"; import { GeneralSettingsSection } from "@/components/admin/general-settings-section"; @@ -166,10 +167,12 @@ export function AdminSectionRenderer({ ); case "backups": - // Two cards, two independent failure modes: the copies held in this - // database, then the copies that reached the operator's bucket. + // The verdict across every backup first, then two cards, two + // independent failure modes: the copies held in this database, then + // the copies that reached the operator's bucket. return ( + diff --git a/src/app/admin/page.tsx b/src/app/admin/page.tsx index cb2e9567c..7bcc5f60d 100644 --- a/src/app/admin/page.tsx +++ b/src/app/admin/page.tsx @@ -4,6 +4,7 @@ import { useAuth } from "@/hooks/use-auth"; import { useTranslations } from "@/lib/i18n/context"; import { QueryErrorRow } from "@/components/ui/query-error-row"; import { AdminShell } from "@/components/admin/admin-shell"; +import { RecoveryReadinessCard } from "@/components/admin/recovery-readiness-card"; import { SystemStatusSummary } from "@/components/admin/system-status-summary"; import { VersionTileSection } from "@/components/admin/version-tile-section"; import { RecentAuditPreview } from "@/components/admin/recent-audit-preview"; @@ -49,6 +50,9 @@ export default function AdminOverviewPage() { + {/* v1.46 — every backup signal, the recovery kit included. */} + + diff --git a/src/app/api/admin/recovery-readiness/route.ts b/src/app/api/admin/recovery-readiness/route.ts new file mode 100644 index 000000000..7c34286f9 --- /dev/null +++ b/src/app/api/admin/recovery-readiness/route.ts @@ -0,0 +1,24 @@ +/** + * GET /api/admin/recovery-readiness + * + * Every backup signal with its verdict, its time and where to fix it: the + * database dump, the weekly copy, the off-host copies, the last restore drill + * (no time window: a failed drill stays failed until a later one passes), the + * off-host setup and the recovery kit. The same judgement the hourly + * evaluation pushes to the admins (`src/lib/recovery/readiness-model.ts`). + * Cookie-only admin. + */ +import { apiHandler, requireAdmin } from "@/lib/api-handler"; +import { apiSuccess } from "@/lib/api-response"; +import { annotate } from "@/lib/logging/context"; +import { readRecoveryReadiness } from "@/lib/recovery/readiness"; + +export const dynamic = "force-dynamic"; + +export const GET = apiHandler(async () => { + await requireAdmin(); + annotate({ action: { name: "admin.recoveryReadiness.status" } }); + const readiness = await readRecoveryReadiness(); + annotate({ meta: { recovery_readiness_state: readiness.state } }); + return apiSuccess(readiness); +}); diff --git a/src/app/api/notifications/preferences/route.ts b/src/app/api/notifications/preferences/route.ts index ad39a9675..2a05c0a13 100644 --- a/src/app/api/notifications/preferences/route.ts +++ b/src/app/api/notifications/preferences/route.ts @@ -9,7 +9,10 @@ import { sanitiseZodIssues, } from "@/lib/api-response"; import { notificationPreferenceSchema } from "@/lib/validations/notifications"; -import { EVENT_TYPES, CHANNEL_TYPE_LABELS } from "@/lib/notifications/types"; +import { + CHANNEL_TYPE_LABELS, + eventTypesForRole, +} from "@/lib/notifications/types"; import type { ChannelType } from "@/lib/notifications/types"; import { decrypt, encrypt } from "@/lib/crypto"; import { resolveEventDefaults } from "@/lib/notifications/event-defaults"; @@ -92,7 +95,9 @@ export const GET = apiHandler(async () => { return apiSuccess({ channels: channelData, preferences, - eventTypes: EVENT_TYPES as unknown as string[], + // Admin-only events (backup health) are listed for admins only: nobody + // else is ever sent one, so a switch for it would switch nothing. + eventTypes: eventTypesForRole(user.role), // What an event does on a channel with no stored row, resolved for this // account by the same function the dispatcher asks. A client renders a // switch without a row from this map, never from its own copy of the diff --git a/src/components/admin/__tests__/backups-restore-preview.test.tsx b/src/components/admin/__tests__/backups-restore-preview.test.tsx index 4107fe523..ad65daf0e 100644 --- a/src/components/admin/__tests__/backups-restore-preview.test.tsx +++ b/src/components/admin/__tests__/backups-restore-preview.test.tsx @@ -53,7 +53,7 @@ vi.mock("@tanstack/react-query", () => ({ schedule: { lastSuccessAt: "2026-09-01T03:00:00.000Z", lastSuccessAgeDays: 2, - staleAfterDays: 10, + staleAfterDays: 9, stale: false, lastRun: null, lastRunFailed: false, diff --git a/src/components/admin/__tests__/backups-section-empty.test.tsx b/src/components/admin/__tests__/backups-section-empty.test.tsx index b1d44b41d..a4ab51b2c 100644 --- a/src/components/admin/__tests__/backups-section-empty.test.tsx +++ b/src/components/admin/__tests__/backups-section-empty.test.tsx @@ -6,7 +6,7 @@ import en from "../../../../messages/en.json"; const healthySchedule = { lastSuccessAt: "2026-09-01T03:00:00.000Z", lastSuccessAgeDays: 2, - staleAfterDays: 10, + staleAfterDays: 9, stale: false, lastRun: { at: "2026-09-01T03:00:12.000Z", diff --git a/src/components/admin/__tests__/recovery-readiness-card.test.tsx b/src/components/admin/__tests__/recovery-readiness-card.test.tsx new file mode 100644 index 000000000..3d130ed3b --- /dev/null +++ b/src/components/admin/__tests__/recovery-readiness-card.test.tsx @@ -0,0 +1,154 @@ +/** + * The recovery readiness card. + * + * 1. every signal renders as its own row with the server's state, its + * sentence and its time; + * 2. a row that needs work links to where it is fixed, unless that is the + * page already open; an ok or inactive row carries no link; + * 3. a grace period shows its due date rather than a time of an event; + * 4. a failed drill shows its reason with no time window: a failure months + * old is still on the card; + * 5. the overall badge carries the worst state; the copy resolves in German. + * + * Mutation check: show the fix link on every row and case 2 goes red; drop + * the `detail` line and case 4 goes red. + */ +import { describe, expect, it, vi } from "vitest"; +import { renderToStaticMarkup } from "react-dom/server"; +import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; + +import { I18nProvider } from "@/lib/i18n/context"; +import { queryKeys } from "@/lib/query-keys"; +import { RecoveryReadinessCard } from "@/components/admin/recovery-readiness-card"; +import type { + ReadinessSignal, + RecoveryReadiness, +} from "@/components/admin/use-recovery-readiness"; +import { READINESS_FIX_PATHS } from "@/lib/recovery/readiness-model"; +import { primeLocales } from "@/__tests__/helpers/prime-locales"; + +primeLocales("de"); + +const pathname = vi.hoisted(() => ({ current: "/admin" })); +vi.mock("next/navigation", () => ({ + useRouter: () => ({ push: vi.fn(), replace: vi.fn() }), + usePathname: () => pathname.current, +})); + +function sig( + id: ReadinessSignal["id"], + state: ReadinessSignal["state"], + reason: string, + over: Partial = {}, +): ReadinessSignal { + return { + id, + state, + reason, + at: null, + params: {}, + detail: null, + fixPath: READINESS_FIX_PATHS[id], + ...over, + }; +} + +const READINESS: RecoveryReadiness = { + state: "failing", + evaluatedAt: "2026-10-25T12:00:00.000Z", + signals: [ + sig("database_dump", "inactive", "not_available"), + sig("weekly_copy", "attention", "stale", { + at: "2026-10-14T03:00:00.000Z", + params: { days: 11, accounts: 2 }, + }), + sig("offhost_copies", "ok", "fresh", { at: "2026-10-25T02:40:00.000Z" }), + sig("restore_drill", "failing", "failed", { + at: "2026-07-01T04:20:00.000Z", + params: { failed: 1, checked: 4 }, + detail: "1 of 4 backup objects failed. Restore drill: bad tag", + }), + sig("offhost_setup", "ok", "configured"), + sig("recovery_kit", "ok", "not_confirmed_grace", { + at: "2026-10-30T09:00:00.000Z", + params: { keys: "v2, API_TOKEN_HMAC_KEY", count: 2 }, + }), + ], +}; + +function render(readiness: RecoveryReadiness, locale: "en" | "de" = "en") { + const client = new QueryClient(); + client.setQueryData(queryKeys.adminRecoveryReadiness(), readiness); + return renderToStaticMarkup( + + + + + , + ); +} + +/** The markup of one signal's row. */ +function row(html: string, id: string): string { + const start = html.indexOf(`data-testid="recovery-signal-${id}"`); + expect(start).toBeGreaterThan(-1); + const end = html.indexOf("", start); + return html.slice(start, end); +} + +describe("RecoveryReadinessCard", () => { + it("renders every signal with its state and sentence", () => { + pathname.current = "/admin"; + const html = render(READINESS); + for (const s of READINESS.signals) { + expect(row(html, s.id)).toContain(`data-state="${s.state}"`); + } + expect(row(html, "weekly_copy")).toContain( + "The oldest weekly copy is 11 days old. Accounts behind: 2.", + ); + expect(row(html, "database_dump")).toContain( + "This version does not schedule a database dump.", + ); + }); + + it("links a row that needs work to where it is fixed, and nothing else", () => { + pathname.current = "/admin"; + const html = render(READINESS); + expect(row(html, "weekly_copy")).toContain('href="/admin/backups"'); + expect(row(html, "restore_drill")).toContain('href="/admin/backups"'); + expect(row(html, "recovery_kit")).toContain('href="/admin/encryption"'); + expect(row(html, "offhost_copies")).not.toContain("href="); + expect(row(html, "database_dump")).not.toContain("href="); + }); + + it("does not link to the page already open", () => { + pathname.current = "/admin/backups"; + const html = render(READINESS); + expect(row(html, "weekly_copy")).not.toContain("href="); + expect(row(html, "recovery_kit")).toContain('href="/admin/encryption"'); + }); + + it("shows a grace period's due date", () => { + pathname.current = "/admin"; + expect(row(render(READINESS), "recovery_kit")).toContain("Due by "); + }); + + it("keeps a months-old drill failure on the card with its reason", () => { + pathname.current = "/admin"; + const drill = row(render(READINESS), "restore_drill"); + expect(drill).toContain( + "The last restore drill failed: 1 of 4 copies could not be read back.", + ); + expect(drill).toContain("Restore drill: bad tag"); + expect(drill).toContain("Last drill: "); + }); + + it("carries the worst state on the badge, and speaks German", () => { + pathname.current = "/admin"; + const html = render(READINESS); + expect(html).toContain('data-testid="recovery-readiness-state"'); + expect(html).toContain('data-state="failing"'); + expect(render({ ...READINESS, state: "ok" })).toContain(">Ready<"); + expect(render(READINESS, "de")).toContain("Wiederherstellbarkeit"); + }); +}); diff --git a/src/components/admin/recovery-readiness-card.tsx b/src/components/admin/recovery-readiness-card.tsx new file mode 100644 index 000000000..19cf4e5db --- /dev/null +++ b/src/components/admin/recovery-readiness-card.tsx @@ -0,0 +1,208 @@ +"use client"; + +/** + * `` — every backup signal on one card: the database + * dump, the weekly account copies, the off-host copies, the last restore + * drill, the off-host setup and the recovery kit. Each row says its verdict, + * the time it is about and where to fix it. + * + * No time window anywhere: a failed drill stays on this card until a later + * drill passes, where the failing-jobs card forgot it after 72 hours. The + * verdicts are the server's (`src/lib/recovery/readiness-model.ts`), the same + * ones the hourly evaluation pushes to every admin, so the card and the push + * cannot disagree. + * + * Shown on the admin overview and on the Backups page. A row whose fix lives + * on the page already open carries no link to itself. + */ + +import Link from "next/link"; +import { usePathname } from "next/navigation"; +import { + AlertTriangle, + CheckCircle2, + Clock, + LifeBuoy, + Loader2, + MinusCircle, + XCircle, +} from "lucide-react"; + +import { SettingsCardHeader } from "@/components/settings/_card-header"; +import { SettingsCard } from "@/components/settings/settings-card"; +import { Badge } from "@/components/ui/badge"; +import { InlineAction } from "@/components/ui/inline-action"; +import { QueryErrorRow } from "@/components/ui/query-error-row"; +import { useFormatters, useTranslations } from "@/lib/i18n/context"; +import { cn } from "@/lib/utils"; + +import { + useRecoveryReadiness, + type ReadinessSignal, + type ReadinessState, + type RecoveryReadiness, +} from "./use-recovery-readiness"; + +const STATE_ICON: Record< + ReadinessState, + React.ComponentType<{ className?: string; "aria-hidden"?: boolean }> +> = { + ok: CheckCircle2, + attention: AlertTriangle, + failing: XCircle, + inactive: MinusCircle, +}; + +const STATE_TONE: Record = { + ok: "text-success", + attention: "text-warning", + failing: "text-destructive", + inactive: "text-muted-foreground", +}; + +/** A grace period is ok for the verdict, but still a step for the admin. */ +const isGrace = (signal: ReadinessSignal) => signal.reason.endsWith("_grace"); + +function OverallBadge({ + state, +}: { + state: Exclude; +}) { + const { t } = useTranslations(); + const label = { + ok: t("admin.recoveryReadiness.state.ok"), + attention: t("admin.recoveryReadiness.state.attention"), + failing: t("admin.recoveryReadiness.state.failing"), + }[state]; + return ( + + {label} + + ); +} + +function SignalRow({ + signal, + currentPath, +}: { + signal: ReadinessSignal; + currentPath: string | null; +}) { + const { t } = useTranslations(); + const fmt = useFormatters(); + const grace = isGrace(signal); + const Icon = grace ? Clock : STATE_ICON[signal.state]; + const tone = grace ? "text-info" : STATE_TONE[signal.state]; + const date = signal.at ? fmt.dateTime(new Date(signal.at)) : null; + const showFix = + signal.state !== "inactive" && + (signal.state !== "ok" || grace) && + signal.fixPath !== currentPath; + + return ( +
  • + +
    +

    + {t(`admin.recoveryReadiness.signal.${signal.id}`)} +

    +

    + {t( + `admin.recoveryReadiness.reason.${signal.id}.${signal.reason}`, + signal.params, + )} +

    + {date ? ( +

    + {grace + ? t("admin.recoveryReadiness.dueBy", { date }) + : t(`admin.recoveryReadiness.at.${signal.id}`, { date })} +

    + ) : null} + {signal.detail && signal.state !== "ok" ? ( +

    + {signal.detail} +

    + ) : null} +
    + {showFix ? ( + + + {t(`admin.recoveryReadiness.fix.${signal.id}`)} + + + ) : null} +
  • + ); +} + +/** The list itself, for a readiness that has loaded. */ +export function RecoveryReadinessList({ + readiness, + currentPath, +}: { + readiness: RecoveryReadiness; + currentPath: string | null; +}) { + return ( +
      + {readiness.signals.map((signal) => ( + + ))} +
    + ); +} + +export function RecoveryReadinessCard() { + const { t } = useTranslations(); + const currentPath = usePathname(); + const { data, isError, error, refetch } = useRecoveryReadiness(); + + return ( + + : null} + /> + {data ? ( + + ) : isError ? ( + void refetch()} + /> + ) : ( +
    +
    + )} +
    + ); +} diff --git a/src/components/admin/use-recovery-readiness.ts b/src/components/admin/use-recovery-readiness.ts new file mode 100644 index 000000000..74b6ed119 --- /dev/null +++ b/src/components/admin/use-recovery-readiness.ts @@ -0,0 +1,24 @@ +"use client"; + +import { useQuery } from "@tanstack/react-query"; +import { apiGet } from "@/lib/api/api-fetch"; +import { queryKeys } from "@/lib/query-keys"; +import type { RecoveryReadiness } from "@/lib/recovery/readiness-model"; + +export type { + ReadinessSignal, + ReadinessState, + RecoveryReadiness, +} from "@/lib/recovery/readiness-model"; + +/** `GET /api/admin/recovery-readiness`; admin-only and cookie-only. */ +export function useRecoveryReadiness() { + return useQuery({ + queryKey: queryKeys.adminRecoveryReadiness(), + queryFn: () => + apiGet("/api/admin/recovery-readiness", { + credentials: "include", + }), + staleTime: 60_000, + }); +} diff --git a/src/lib/jobs/__tests__/backup-schedule-status.test.ts b/src/lib/jobs/__tests__/backup-schedule-status.test.ts index 2a45bee31..6beaad532 100644 --- a/src/lib/jobs/__tests__/backup-schedule-status.test.ts +++ b/src/lib/jobs/__tests__/backup-schedule-status.test.ts @@ -50,6 +50,19 @@ describe("summariseBackupSchedule", () => { expect(status.lastRunFailed).toBe(false); }); + it("draws the line where the readiness model does: past nine days", () => { + const at = (ms: number) => + summariseBackupSchedule({ + scheduledCreatedAt: [new Date(NOW.getTime() - ms)], + lastRun: null, + now: NOW, + }).stale; + const DAY = 24 * 60 * 60 * 1000; + expect(BACKUP_STALE_AFTER_DAYS).toBe(9); + expect(at(9 * DAY)).toBe(false); + expect(at(9 * DAY + 1)).toBe(true); + }); + it("surfaces a failed last run even while the copy is still in window", () => { const status = summariseBackupSchedule({ scheduledCreatedAt: [daysAgo(3)], diff --git a/src/lib/jobs/__tests__/restore-drill.test.ts b/src/lib/jobs/__tests__/restore-drill.test.ts index 07136af8d..e6fe03ea3 100644 --- a/src/lib/jobs/__tests__/restore-drill.test.ts +++ b/src/lib/jobs/__tests__/restore-drill.test.ts @@ -14,7 +14,12 @@ vi.mock("../report-worker-error", () => ({ reportWorkerError: vi.fn().mockResolvedValue(undefined), })); +vi.mock("../restore-drill-outcome", () => ({ + recordRestoreDrillOutcome: vi.fn().mockResolvedValue(undefined), +})); + import { reportWorkerError } from "../report-worker-error"; +import { recordRestoreDrillOutcome } from "../restore-drill-outcome"; const ENC_KEY = "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"; @@ -257,6 +262,8 @@ describe("handleRestoreDrill", () => { vi.stubEnv("BACKUP_S3_ENDPOINT", ""); await handleRestoreDrill([]); expect(reportWorkerError).not.toHaveBeenCalled(); + // Nothing was checked, so there is no verdict to keep. + expect(recordRestoreDrillOutcome).not.toHaveBeenCalled(); }); it("pages via reportWorkerError when the drill fails", async () => { @@ -270,5 +277,22 @@ describe("handleRestoreDrill", () => { expect(vi.mocked(reportWorkerError).mock.calls[0][0]).toBe( RESTORE_DRILL_QUEUE, ); + // The failure is kept for the readiness tile, past pg-boss's retention. + expect(recordRestoreDrillOutcome).toHaveBeenCalledTimes(1); + expect(vi.mocked(recordRestoreDrillOutcome).mock.calls[0][0]).toMatchObject( + { ok: false, accountsChecked: 0, accountsFailed: 0 }, + ); + }); + + it("keeps a failed drill's verdict even when keeping it fails", async () => { + stubConfigEnv(); + vi.stubEnv("BACKUP_ENCRYPTION_KEY", "not-a-key"); + vi.mocked(recordRestoreDrillOutcome).mockRejectedValueOnce( + new Error("database away"), + ); + const outcome = await handleRestoreDrill([]); + // The drill's own failure is still what the run reports. + expect(outcome.ok).toBe(false); + expect(reportWorkerError).toHaveBeenCalledTimes(1); }); }); diff --git a/src/lib/jobs/backup-schedule-status.ts b/src/lib/jobs/backup-schedule-status.ts index d7a5ac20d..38264c08a 100644 --- a/src/lib/jobs/backup-schedule-status.ts +++ b/src/lib/jobs/backup-schedule-status.ts @@ -23,13 +23,15 @@ * alive. */ import type { LastQueueRun } from "@/lib/jobs/job-failures"; +import { WEEKLY_COPY_ATTENTION_AFTER_DAYS } from "@/lib/recovery/readiness-model"; /** * How old the newest scheduled copy may get before the page says so. The cron - * is weekly, so seven days is normal and eight is a rounding error; ten means - * at least one Sunday produced nothing. + * is weekly, so seven days is normal and eight is a long week; past nine at + * least one Sunday produced nothing. The recovery readiness model draws its + * attention line at the same instant, so this page and the push agree. */ -export const BACKUP_STALE_AFTER_DAYS = 10; +export const BACKUP_STALE_AFTER_DAYS = WEEKLY_COPY_ATTENTION_AFTER_DAYS; const DAY_MS = 24 * 60 * 60 * 1000; @@ -71,7 +73,9 @@ export function summariseBackupSchedule(args: { // new — the page's empty state already says there is nothing there, and // calling a fresh install "overdue" would train the operator to ignore the // warning that matters. - stale: ageDays !== null && ageDays > BACKUP_STALE_AFTER_DAYS, + stale: + newest !== null && + args.now.getTime() - newest.getTime() > BACKUP_STALE_AFTER_DAYS * DAY_MS, lastRun: args.lastRun, lastRunFailed: args.lastRun !== null && args.lastRun.state !== "completed", }; diff --git a/src/lib/jobs/job-outcome.ts b/src/lib/jobs/job-outcome.ts index 250838b7d..007d7c6f7 100644 --- a/src/lib/jobs/job-outcome.ts +++ b/src/lib/jobs/job-outcome.ts @@ -234,6 +234,13 @@ export const JOB_FACT_ALLOWLIST: ReadonlySet = new Set([ "restore_drill_plaintext_bytes", "restore_drill_unchecked", "restore_drill_stale", + // The hourly recovery readiness evaluation: the overall verdict, how many + // signals need attention or are failing, notices sent, admins reached. + "recovery_readiness_admins", + "recovery_readiness_attention", + "recovery_readiness_failing", + "recovery_readiness_notices", + "recovery_readiness_state", "restore_fail_cause", "restore_failed", "restore_measurements", diff --git a/src/lib/jobs/queue-runtime.ts b/src/lib/jobs/queue-runtime.ts index 28ed26810..0f0c273d3 100644 --- a/src/lib/jobs/queue-runtime.ts +++ b/src/lib/jobs/queue-runtime.ts @@ -368,6 +368,9 @@ export const QUEUE_RUNTIME: Readonly> = { "geo-backfill": short("At most 500 audit rows per pass."), "geolite2-fetch": short("One database download."), "tls-pin-monitor": short("One TLS handshake."), + "recovery-readiness": short( + "A few indexed reads, one row per account from two backup ledgers, and at most one notice per signal.", + ), "integration-watchdog-hourly": short( "A few indexed reads per account with a connection and no provider call; a 14-day history count only for a silent source.", ), diff --git a/src/lib/jobs/recovery-readiness-queue.ts b/src/lib/jobs/recovery-readiness-queue.ts new file mode 100644 index 000000000..77e1f3b74 --- /dev/null +++ b/src/lib/jobs/recovery-readiness-queue.ts @@ -0,0 +1,12 @@ +/** + * The queue facts of `recovery-readiness.ts`: its name and schedule. They live + * apart from the job so the worker registry can read them without loading it. + */ +export const RECOVERY_READINESS_QUEUE = "recovery-readiness"; + +/** + * Hourly at :33, a minute no other schedule uses. Hourly is far inside every + * threshold the evaluation judges (36 hours for the dump, two nights for the + * off-host copies), and a notice is sent once per change whatever the cadence. + */ +export const RECOVERY_READINESS_CRON = "33 * * * *"; diff --git a/src/lib/jobs/recovery-readiness.ts b/src/lib/jobs/recovery-readiness.ts new file mode 100644 index 000000000..18518d330 --- /dev/null +++ b/src/lib/jobs/recovery-readiness.ts @@ -0,0 +1,42 @@ +/** + * The hourly recovery readiness evaluation: read every backup signal, judge + * it, and push each change to the admins (`readiness-notify.ts`). The tile on + * the admin pages reads the same verdict live; this job is what makes it a + * push rather than a page somebody has to open. + * + * Cheap and bounded: a handful of indexed reads, one row per account for the + * two backup ledgers, and at most one notice per signal per run. The next + * tick is the retry. + */ +import type { Job } from "pg-boss"; + +import { jobDone, jobFailed, type JobOutcome } from "@/lib/jobs/job-outcome"; +import { withBackgroundEvent } from "@/lib/logging/background"; +import { readRecoveryReadiness } from "@/lib/recovery/readiness"; +import { notifyReadinessChanges } from "@/lib/recovery/readiness-notify"; + +export async function handleRecoveryReadiness( + jobs: Job[], +): Promise { + void jobs; + return withBackgroundEvent("job.recovery_readiness", async (evt) => { + try { + const readiness = await readRecoveryReadiness(); + const outcome = await notifyReadinessChanges(readiness); + const counts = (state: string) => + readiness.signals.filter((s) => s.state === state).length; + const did = { + recovery_readiness_state: readiness.state, + recovery_readiness_attention: counts("attention"), + recovery_readiness_failing: counts("failing"), + recovery_readiness_notices: outcome.sent.length, + recovery_readiness_admins: outcome.admins, + }; + for (const [key, value] of Object.entries(did)) evt.addMeta(key, value); + return jobDone(did); + } catch (err) { + evt.addWarning(`recovery-readiness failed: ${err}`); + return jobFailed("recovery readiness evaluation failed", err); + } + }); +} diff --git a/src/lib/jobs/registry/maintenance.ts b/src/lib/jobs/registry/maintenance.ts index 70429c156..a13bb2dd2 100644 --- a/src/lib/jobs/registry/maintenance.ts +++ b/src/lib/jobs/registry/maintenance.ts @@ -169,6 +169,10 @@ import { } from "@/lib/jobs/pr-detection"; import { PULSE_DAY_MEAN_QUEUE } from "@/lib/jobs/pulse-day-mean"; import { cronIsTheRetry } from "@/lib/jobs/reminder/registrar-shared"; +import { + RECOVERY_READINESS_CRON, + RECOVERY_READINESS_QUEUE, +} from "@/lib/jobs/recovery-readiness-queue"; import { RESTORE_DRILL_CRON, RESTORE_DRILL_QUEUE, @@ -343,6 +347,17 @@ const jobs: readonly JobDefinition[] = [ load: async () => (await import("@/lib/jobs/restore-drill")).handleRestoreDrill, }), + // Hourly: judge every backup signal and push each change to the admins. + // Single-flight and no retries; the next tick is the retry, and the notice + // claim keeps two evaluations from sending the same notice. + defineJob({ + queue: RECOVERY_READINESS_QUEUE, + cron: RECOVERY_READINESS_CRON, + scheduleOptions: cronIsTheRetry, + work: { localConcurrency: 1 }, + load: async () => + (await import("@/lib/jobs/recovery-readiness")).handleRecoveryReadiness, + }), // The nightly backstop for the purge an account deletion kicks at once. prune( OFFHOST_PURGE_QUEUE, diff --git a/src/lib/jobs/restore-drill-outcome.ts b/src/lib/jobs/restore-drill-outcome.ts new file mode 100644 index 000000000..c6ab0bb69 --- /dev/null +++ b/src/lib/jobs/restore-drill-outcome.ts @@ -0,0 +1,54 @@ +/** + * The monthly restore drill's verdict, kept until the next drill replaces it. + * + * The drill used to say what it found only through `reportWorkerError` + * (stderr, and GlitchTip where configured) and through the pg-boss row of its + * run. The row is gone after seven days and the admin failing-jobs card reads + * 72 hours of it, so a drill that failed on the first of the month was on + * screen until the fourth and invisible for the rest of it. One row, written + * at the end of every drill that had something to check, is what the recovery + * readiness tile and its push read instead, with no window at all. + * + * A failure stays the verdict until a later drill passes: `lastFailureAt` + * newer than `lastSuccessAt` is the rule (`evaluateRestoreDrill`). + */ +import { prisma } from "@/lib/db"; +import { redactSecrets } from "@/lib/logging/redact"; + +/** The reason is a line on a status tile, not a stack trace. */ +const MAX_REASON_CHARS = 300; + +export interface RestoreDrillResult { + ok: boolean; + accountsChecked: number; + accountsFailed: number; + /** Why it failed; ignored when `ok`. */ + reason?: string | null; +} + +export async function recordRestoreDrillOutcome( + result: RestoreDrillResult, + at: Date = new Date(), +): Promise { + const reason = result.ok + ? null + : (() => { + const text = redactSecrets((result.reason ?? "").trim()); + return text.length > MAX_REASON_CHARS + ? `${text.slice(0, MAX_REASON_CHARS)}…` + : text || null; + })(); + const verdict = result.ok + ? { lastSuccessAt: at } + : { lastFailureAt: at, lastFailureReason: reason }; + const counts = { + lastRunAt: at, + accountsChecked: result.accountsChecked, + accountsFailed: result.accountsFailed, + }; + await prisma.restoreDrillOutcome.upsert({ + where: { id: "singleton" }, + create: { id: "singleton", ...counts, ...verdict }, + update: { ...counts, ...verdict }, + }); +} diff --git a/src/lib/jobs/restore-drill.ts b/src/lib/jobs/restore-drill.ts index a2d38c476..deb8d1353 100644 --- a/src/lib/jobs/restore-drill.ts +++ b/src/lib/jobs/restore-drill.ts @@ -15,8 +15,10 @@ * application key it needs. Each account is reported on its own. It deliberately performs NO database restore — the * drill validates the artefact, not the import path. The result is * surfaced through the wide-event meta on success and through - * `reportWorkerError` (stderr + GlitchTip) on any failure, so a broken - * backup chain pages the operator eleven months before it matters. + * `reportWorkerError` (stderr + GlitchTip) on any failure, and the verdict is + * kept (`restore-drill-outcome.ts`) for the recovery readiness tile and its + * push to the admins, so a broken backup chain reaches the operator eleven + * months before it matters. * * Schedule: 04:11 on the 1st of each month (`11 4 1 * *`), after the * 02:30 nightly upload and the 03:xx cleanup window, on a minute slot @@ -45,6 +47,10 @@ import { type S3Like, } from "@/lib/jobs/offhost-backup"; import { reportWorkerError } from "@/lib/jobs/report-worker-error"; +import { + recordRestoreDrillOutcome, + type RestoreDrillResult, +} from "@/lib/jobs/restore-drill-outcome"; import { jobBudget } from "@/lib/jobs/job-budget"; import { jobDone, jobFailed, type JobOutcome } from "@/lib/jobs/job-outcome"; import { withBackgroundEvent } from "@/lib/logging/background"; @@ -254,6 +260,21 @@ export async function runRestoreDrill( }; } +/** + * Keep the verdict for the readiness tile. A failed write is a warning on the + * run, never a failed drill: the drill's own reports already went out. + */ +async function keepOutcome( + evt: { addWarning: (message: string) => void }, + result: RestoreDrillResult, +): Promise { + try { + await recordRestoreDrillOutcome(result); + } catch (err) { + evt.addWarning(`restore-drill outcome not recorded: ${err}`); + } +} + /** * pg-boss handler. Mirrors the off-host uploader's posture: a * not-configured deployment skips with a wide-event warning (most @@ -323,6 +344,17 @@ export async function handleRestoreDrill( { dateKey: report.dateKey, unchecked: report.unchecked }, ); } + await keepOutcome(evt, { + ok: report.failed.length === 0, + accountsChecked: report.accounts.length, + accountsFailed: report.failed.length, + reason: + report.failed.length === 0 + ? null + : `${report.failed.length} of ${report.accounts.length} backup objects failed. ${ + report.failed[0].error ?? report.failed[0].objectKey + }`, + }); if (report.failed.length > 0) { // Already paged above, per account. The queue does not retry: a // retry reads the same objects and would page the same accounts again. @@ -346,6 +378,12 @@ export async function handleRestoreDrill( } evt.addWarning(`restore-drill failed: ${err}`); await reportWorkerError(RESTORE_DRILL_QUEUE, err); + await keepOutcome(evt, { + ok: false, + accountsChecked: 0, + accountsFailed: 0, + reason: err instanceof Error ? err.message : String(err), + }); return jobFailed("restore drill failed", err); } }); diff --git a/src/lib/jobs/subsystem-surface.ts b/src/lib/jobs/subsystem-surface.ts index d90560c84..a2cd5a468 100644 --- a/src/lib/jobs/subsystem-surface.ts +++ b/src/lib/jobs/subsystem-surface.ts @@ -100,6 +100,7 @@ const subsystemSurface = { "step-up-elevation-cleanup": { audience: "system" }, "data-backup-offhost": { audience: "system" }, "data-restore-drill": { audience: "system" }, + "recovery-readiness": { audience: "system" }, "offhost-backup-purge": { audience: "system" }, "backup-restore": { audience: "system" }, "host-metric-sample": { audience: "system" }, diff --git a/src/lib/notifications/tap-target.ts b/src/lib/notifications/tap-target.ts index 95fd59c40..60e1ac7c1 100644 --- a/src/lib/notifications/tap-target.ts +++ b/src/lib/notifications/tap-target.ts @@ -51,6 +51,7 @@ export const NOTIFICATION_TAP_PATHS: Record = { SECURITY_ALERT: "/settings/security", DAILY_BRIEFING: "/", SHARED_RECORD_INTAKE: "/medications", + BACKUP_HEALTH: "/admin/backups", }; const MAX_PATH_LENGTH = 512; diff --git a/src/lib/notifications/types.ts b/src/lib/notifications/types.ts index a84a01395..ea05bf42a 100644 --- a/src/lib/notifications/types.ts +++ b/src/lib/notifications/types.ts @@ -125,6 +125,13 @@ export const EVENT_TYPES = [ // dispatcher addresses a userId, and under record substitution that userId // is the owner by design). Default ON, see EVENT_DEFAULT_ENABLED. "SHARED_RECORD_INTAKE", + // Recovery readiness, for admins only: a backup signal turned to + // attention or failing, is still failing a day later, or is back to normal + // (`src/lib/recovery/readiness-notify.ts`). Default ON: the whole point is + // that a stopped backup is pushed rather than found on a page weeks later. + // An admin routes it per channel in the /notifications matrix, where only + // admins see the row (`eventTypesForRole`). + "BACKUP_HEALTH", ] as const; export type EventType = (typeof EVENT_TYPES)[number]; @@ -214,8 +221,21 @@ export const EVENT_DEFAULT_ENABLED: Record = { // sits in the /notifications matrix from day one, so an owner who finds it // noisy silences it per channel with one switch. SHARED_RECORD_INTAKE: true, + // ON by default; see the note on `EVENT_TYPES`. Only admins are ever + // addressed, and at most once per state change per signal. + BACKUP_HEALTH: true, }; +/** Event types only an admin is ever sent, and only an admin is shown. */ +export const ADMIN_ONLY_EVENT_TYPES: readonly EventType[] = ["BACKUP_HEALTH"]; + +/** The event types a person with `role` can receive, in matrix order. */ +export function eventTypesForRole(role: string): EventType[] { + return role === "ADMIN" + ? [...EVENT_TYPES] + : EVENT_TYPES.filter((type) => !ADMIN_ONLY_EVENT_TYPES.includes(type)); +} + export const CHANNEL_TYPE_LABELS: Record = { TELEGRAM: "Telegram", NTFY: "ntfy", diff --git a/src/lib/recovery/__tests__/readiness-model.test.ts b/src/lib/recovery/__tests__/readiness-model.test.ts new file mode 100644 index 000000000..dc15c3bf0 --- /dev/null +++ b/src/lib/recovery/__tests__/readiness-model.test.ts @@ -0,0 +1,424 @@ +/** + * The recovery readiness thresholds, one signal at a time, on both sides of + * every line the model draws. + * + * Mutation check: move any threshold by one unit (35 h for the dump, 8 or 13 + * days for the weekly copy, 6 days of setup grace) and the matching + * "just inside" or "just past" case goes red. + */ +import { describe, expect, it } from "vitest"; + +import type { DatabaseBackupHealth } from "@/lib/recovery/database-dump-health"; +import { + DUMP_FAILING_AFTER_HOURS, + evaluateDatabaseDump, + evaluateOffhostCopies, + evaluateOffhostSetup, + evaluateRecoveryKit, + evaluateRecoveryReadiness, + evaluateRestoreDrill, + evaluateWeeklyCopy, + READINESS_FIX_PATHS, + READINESS_SIGNALS, + SETUP_GRACE_DAYS, + WEEKLY_COPY_ATTENTION_AFTER_DAYS, + WEEKLY_COPY_FAILING_AFTER_DAYS, + type ReadinessInputs, +} from "@/lib/recovery/readiness-model"; + +const NOW = new Date("2026-10-25T12:00:00Z"); +const HOUR = 3_600_000; +const DAY = 24 * HOUR; +const ago = (ms: number) => new Date(NOW.getTime() - ms); + +/** A dump report in the dump module's own shape. */ +function dump( + over: Partial & { + lastGoodAt?: Date | null; + lastFailureAt?: Date | null; + lastFailureReason?: string; + } = {}, +): DatabaseBackupHealth { + const { + lastGoodAt = ago(2 * HOUR), + lastFailureAt = null, + lastFailureReason, + ...rest + } = over; + return { + enabled: true, + directory: "/data/db-backups", + localDirWritable: true, + offhostConfigured: false, + lastGood: lastGoodAt + ? { + at: lastGoodAt.toISOString(), + fileName: "healthlog.dump.enc", + sizeBytes: 1024, + local: true, + offhost: false, + } + : null, + lastFailure: lastFailureAt + ? { + at: lastFailureAt.toISOString(), + error: lastFailureReason ?? "failed", + } + : null, + ...rest, + }; +} + +function inputs(over: Partial = {}): ReadinessInputs { + return { + now: NOW, + firstAdminAt: ago(30 * DAY), + dump: null, + dumpWatchedSince: null, + weekly: { accounts: [{ createdAt: ago(60 * DAY), lastCopyAt: ago(DAY) }] }, + offhost: { + configured: true, + accounts: [ + { lastAttemptAt: ago(10 * HOUR), lastSuccessAt: ago(10 * HOUR) }, + ], + }, + drill: null, + kit: { confirmedAt: ago(10 * DAY), unconfirmed: [] }, + ...over, + }; +} + +describe("database dump", () => { + const evalDump = (d: ReadinessInputs["dump"], watched: Date | null = null) => + evaluateDatabaseDump({ now: NOW, dump: d, dumpWatchedSince: watched }); + + it("does not apply while no dump is scheduled, or while it is switched off", () => { + expect(evalDump(null)).toMatchObject({ + state: "inactive", + reason: "not_available", + }); + expect(evalDump(dump({ enabled: false }))).toMatchObject({ + state: "inactive", + reason: "disabled", + }); + }); + + it("fails when its status cannot be read", () => { + expect(evalDump("unreadable")).toMatchObject({ + state: "failing", + reason: "unreadable", + }); + }); + + it("is ok up to 36 hours and failing past them", () => { + const inside = evalDump( + dump({ lastGoodAt: ago(DUMP_FAILING_AFTER_HOURS * HOUR - 1) }), + ); + expect(inside).toMatchObject({ state: "ok", reason: "fresh" }); + const past = evalDump( + dump({ lastGoodAt: ago(DUMP_FAILING_AFTER_HOURS * HOUR + 1) }), + ); + expect(past).toMatchObject({ + state: "failing", + reason: "stale", + params: { hours: 36 }, + }); + expect(past.at).toBe( + ago(DUMP_FAILING_AFTER_HOURS * HOUR + 1).toISOString(), + ); + }); + + it("calls a recent failure after a recent dump attention, with the reason", () => { + const s = evalDump( + dump({ lastFailureAt: ago(HOUR), lastFailureReason: "pg_dump exited 1" }), + ); + expect(s).toMatchObject({ + state: "attention", + reason: "last_failed", + detail: "pg_dump exited 1", + }); + }); + + it("fails when a dump can be kept nowhere, and accepts a bucket-only setup", () => { + expect(evalDump(dump({ localDirWritable: false }))).toMatchObject({ + state: "failing", + reason: "nowhere_to_write", + params: { directory: "/data/db-backups" }, + }); + expect( + evalDump(dump({ localDirWritable: false, offhostConfigured: true })) + .state, + ).toBe("ok"); + }); + + it("gives a newly switched-on dump 36 hours before calling it failing", () => { + const fresh = dump({ lastGoodAt: null }); + expect(evalDump(fresh, ago(HOUR))).toMatchObject({ + state: "ok", + reason: "first_pending", + }); + expect(evalDump(fresh, null)).toMatchObject({ + state: "ok", + reason: "first_pending", + }); + expect(evalDump(fresh, ago(37 * HOUR))).toMatchObject({ + state: "failing", + reason: "never_ran", + }); + expect( + evalDump(dump({ lastGoodAt: null, lastFailureAt: ago(HOUR) }), ago(HOUR)), + ).toMatchObject({ state: "failing", reason: "never_succeeded" }); + }); +}); + +describe("weekly copy", () => { + const weekly = ( + ...accounts: Array<{ createdAt: Date; lastCopyAt: Date | null }> + ) => evaluateWeeklyCopy({ now: NOW, weekly: { accounts } }); + + it("is attention past 9 days and failing past 14, judged by the worst account", () => { + const fresh = { createdAt: ago(90 * DAY), lastCopyAt: ago(DAY) }; + expect( + weekly(fresh, { createdAt: ago(90 * DAY), lastCopyAt: ago(9 * DAY - 1) }) + .state, + ).toBe("ok"); + expect( + weekly(fresh, { + createdAt: ago(90 * DAY), + lastCopyAt: ago(WEEKLY_COPY_ATTENTION_AFTER_DAYS * DAY + 1), + }), + ).toMatchObject({ + state: "attention", + reason: "stale", + params: { days: 9, accounts: 1 }, + }); + expect( + weekly(fresh, { + createdAt: ago(90 * DAY), + lastCopyAt: ago(WEEKLY_COPY_FAILING_AFTER_DAYS * DAY - 1), + }).state, + ).toBe("attention"); + expect( + weekly(fresh, { + createdAt: ago(90 * DAY), + lastCopyAt: ago(WEEKLY_COPY_FAILING_AFTER_DAYS * DAY + 1), + }), + ).toMatchObject({ state: "failing", params: { days: 14 } }); + }); + + it("ages an account without any copy from its creation", () => { + expect(weekly({ createdAt: ago(3 * DAY), lastCopyAt: null })).toMatchObject( + { + state: "ok", + reason: "first_pending", + }, + ); + expect( + weekly({ createdAt: ago(20 * DAY), lastCopyAt: null }), + ).toMatchObject({ + state: "failing", + reason: "never", + }); + }); + + it("does not apply without accounts", () => { + expect(weekly().state).toBe("inactive"); + }); +}); + +describe("off-host copies", () => { + const offhost = ( + configured: boolean, + ...accounts: Array<{ + lastAttemptAt: Date | null; + lastSuccessAt: Date | null; + }> + ) => evaluateOffhostCopies({ now: NOW, offhost: { configured, accounts } }); + + it("does not apply while no bucket is configured", () => { + expect(offhost(false).state).toBe("inactive"); + }); + + it("calls any account more than two nights behind attention", () => { + const fresh = { + lastAttemptAt: ago(5 * HOUR), + lastSuccessAt: ago(5 * HOUR), + }; + // One missed night (inside two periods plus the grace) is not yet news. + expect( + offhost(true, fresh, { + lastAttemptAt: ago(HOUR), + lastSuccessAt: ago(40 * HOUR), + }).state, + ).toBe("ok"); + expect( + offhost(true, fresh, { + lastAttemptAt: ago(HOUR), + lastSuccessAt: ago(2 * DAY + 7 * HOUR), + }), + ).toMatchObject({ + state: "attention", + reason: "stale", + params: { accounts: 1 }, + }); + }); + + it("counts an account a run reached and never wrote as behind", () => { + expect( + offhost(true, { lastAttemptAt: ago(HOUR), lastSuccessAt: null }), + ).toMatchObject({ state: "attention", params: { accounts: 1 } }); + }); + + it("waits for the first run rather than calling the cohort behind", () => { + expect( + offhost(true, { lastAttemptAt: null, lastSuccessAt: null }), + ).toMatchObject({ + state: "ok", + reason: "first_pending", + }); + }); +}); + +describe("restore drill", () => { + const drill = (d: ReadinessInputs["drill"], configured = true) => + evaluateRestoreDrill({ offhost: { configured, accounts: [] }, drill: d }); + const run = { + lastRunAt: ago(5 * DAY), + lastSuccessAt: null as Date | null, + lastFailureAt: null as Date | null, + lastFailureReason: null as string | null, + accountsChecked: 3, + accountsFailed: 0, + }; + + it("stays failing after a failure, with no time window, until a later success", () => { + const failed = { + ...run, + lastFailureAt: ago(60 * DAY), + accountsFailed: 1, + lastFailureReason: "bad tag", + }; + expect(drill(failed)).toMatchObject({ + state: "failing", + reason: "failed", + params: { failed: 1, checked: 3 }, + detail: "bad tag", + }); + expect(drill({ ...failed, lastSuccessAt: ago(61 * DAY) }).state).toBe( + "failing", + ); + expect(drill({ ...failed, lastSuccessAt: ago(DAY) })).toMatchObject({ + state: "ok", + reason: "passed", + }); + }); + + it("names a run that checked nothing", () => { + expect( + drill({ ...run, accountsChecked: 0, lastFailureAt: ago(DAY) }).reason, + ).toBe("failed_run"); + }); + + it("does not apply without a bucket, and is ok before its first run", () => { + expect(drill(null, false).state).toBe("inactive"); + expect(drill(null)).toMatchObject({ state: "ok", reason: "not_run_yet" }); + }); +}); + +describe("off-host setup", () => { + const setup = (configured: boolean, firstAdminAt: Date | null) => + evaluateOffhostSetup({ + now: NOW, + firstAdminAt, + offhost: { configured, accounts: [] }, + }); + + it("asks for a bucket 7 days after the first admin exists", () => { + expect(setup(false, ago(SETUP_GRACE_DAYS * DAY - 1))).toMatchObject({ + state: "ok", + reason: "not_configured_grace", + }); + expect(setup(false, ago(SETUP_GRACE_DAYS * DAY + 1))).toMatchObject({ + state: "attention", + reason: "not_configured", + }); + expect(setup(true, ago(90 * DAY)).state).toBe("ok"); + expect(setup(false, null).state).toBe("inactive"); + }); +}); + +describe("recovery kit", () => { + const kit = ( + confirmedAt: Date | null, + unconfirmed: Array<{ + kind: "application" | "backup" | "token"; + id: string; + }>, + firstAdminAt: Date | null = ago(30 * DAY), + ) => + evaluateRecoveryKit({ + now: NOW, + firstAdminAt, + kit: { confirmedAt, unconfirmed }, + }); + + it("is ok while the confirmation covers every key", () => { + expect(kit(ago(DAY), [])).toMatchObject({ + state: "ok", + reason: "confirmed", + }); + }); + + it("names a needed key nobody confirmed, at once once a kit was confirmed", () => { + expect(kit(ago(DAY), [{ kind: "application", id: "v1" }])).toMatchObject({ + state: "attention", + reason: "out_of_date", + params: { keys: "v1", count: 1 }, + }); + }); + + it("gives a never-confirmed kit 7 days after setup", () => { + const keys = [ + { kind: "application" as const, id: "v1" }, + { kind: "token" as const, id: "API_TOKEN_HMAC_KEY" }, + ]; + expect(kit(null, keys, ago(SETUP_GRACE_DAYS * DAY - 1))).toMatchObject({ + state: "ok", + reason: "not_confirmed_grace", + }); + expect(kit(null, keys, ago(SETUP_GRACE_DAYS * DAY + 1))).toMatchObject({ + state: "attention", + reason: "not_confirmed", + params: { keys: "v1, API_TOKEN_HMAC_KEY", count: 2 }, + }); + }); +}); + +describe("the whole verdict", () => { + it("is the worst applying signal, and inactive ones never count", () => { + expect(evaluateRecoveryReadiness(inputs()).state).toBe("ok"); + expect( + evaluateRecoveryReadiness( + inputs({ dump: dump({ lastGoodAt: ago(40 * HOUR) }) }), + ).state, + ).toBe("failing"); + expect( + evaluateRecoveryReadiness( + inputs({ + kit: { + confirmedAt: ago(DAY), + unconfirmed: [{ kind: "backup", id: "abc" }], + }, + }), + ).state, + ).toBe("attention"); + }); + + it("reports every signal once, in order, each with a fix path", () => { + const r = evaluateRecoveryReadiness(inputs()); + expect(r.signals.map((s) => s.id)).toEqual([...READINESS_SIGNALS]); + for (const s of r.signals) + expect(s.fixPath).toBe(READINESS_FIX_PATHS[s.id]); + expect(r.evaluatedAt).toBe(NOW.toISOString()); + }); +}); diff --git a/src/lib/recovery/__tests__/readiness-notify.test.ts b/src/lib/recovery/__tests__/readiness-notify.test.ts new file mode 100644 index 000000000..95e0bc949 --- /dev/null +++ b/src/lib/recovery/__tests__/readiness-notify.test.ts @@ -0,0 +1,287 @@ +/** + * The notice state machine: once per state change per signal, a recovery + * notice once, a reminder at most once a day while failing, nothing else. + * + * Two layers. `decideNotice` is pure and pinned case by case. + * `notifyReadinessChanges` runs against an in-memory stand-in for its one + * table that honours the compare-and-set exactly as Postgres would (a row + * moves only from the values the caller read), so the hourly sequence and the + * two-workers race are proven on the real claim logic rather than assumed. + * + * Mutation check: drop the `last !== current` comparison and the "same state + * twice" case sends twice; drop the CAS `where` on `notifiedState` and the + * race sends two notices. + */ +import { beforeEach, describe, expect, it, vi } from "vitest"; + +const store = vi.hoisted(() => ({ + notices: new Map< + string, + { + signal: string; + notifiedState: string | null; + notifiedAt: Date | null; + watchedSince: Date | null; + } + >(), + admins: [] as Array<{ id: string }>, +})); + +const same = (a: Date | null, b: Date | null | undefined) => + (a?.getTime() ?? null) === (b?.getTime() ?? null); + +vi.mock("@/lib/db", () => ({ + prisma: { + user: { findMany: vi.fn(async () => store.admins) }, + recoveryReadinessNotice: { + createMany: vi.fn( + async ({ data }: { data: Array<{ signal: string }> }) => { + for (const { signal } of data) { + if (!store.notices.has(signal)) { + store.notices.set(signal, { + signal, + notifiedState: null, + notifiedAt: null, + watchedSince: null, + }); + } + } + return { count: data.length }; + }, + ), + findMany: vi.fn(async () => + [...store.notices.values()].map((row) => ({ ...row })), + ), + updateMany: vi.fn( + async ({ + where, + data, + }: { + where: { + signal: string; + notifiedState?: string | null; + notifiedAt?: Date | null; + watchedSince?: Date | null; + }; + data: Record; + }) => { + const row = store.notices.get(where.signal); + if (!row) return { count: 0 }; + if ( + "notifiedState" in where && + row.notifiedState !== where.notifiedState + ) + return { count: 0 }; + if ("notifiedAt" in where && !same(row.notifiedAt, where.notifiedAt)) + return { count: 0 }; + if ( + "watchedSince" in where && + !same(row.watchedSince, where.watchedSince) + ) + return { count: 0 }; + Object.assign(row, data); + return { count: 1 }; + }, + ), + }, + }, +})); + +const dispatch = vi.hoisted(() => + vi.fn(async (opts: Record) => { + void opts; + }), +); +vi.mock("@/lib/notifications/dispatch-localised", () => ({ + dispatchLocalisedNotification: dispatch, +})); + +import { + decideNotice, + FAILING_REMINDER_HOURS, + notifyReadinessChanges, +} from "@/lib/recovery/readiness-notify"; +import { + READINESS_FIX_PATHS, + READINESS_SIGNALS, + type ReadinessSignal, + type ReadinessState, + type RecoveryReadiness, +} from "@/lib/recovery/readiness-model"; + +const T0 = new Date("2026-10-25T12:00:00Z"); +const HOUR = 3_600_000; +const at = (hours: number) => new Date(T0.getTime() + hours * HOUR); + +describe("decideNotice", () => { + it("announces a change to attention or failing once", () => { + expect(decideNotice(null, "attention", T0)).toEqual({ + send: "problem", + next: { notifiedState: "attention", notifiedAt: T0 }, + }); + const told = { notifiedState: "attention" as const, notifiedAt: T0 }; + expect(decideNotice(told, "attention", at(30)).send).toBeNull(); + expect(decideNotice(told, "failing", at(1)).send).toBe("problem"); + }); + + it("reminds a failing signal once a day, not sooner", () => { + const told = { notifiedState: "failing" as const, notifiedAt: T0 }; + expect( + decideNotice(told, "failing", at(FAILING_REMINDER_HOURS - 1)).send, + ).toBeNull(); + expect(decideNotice(told, "failing", at(FAILING_REMINDER_HOURS))).toEqual({ + send: "reminder", + next: { + notifiedState: "failing", + notifiedAt: at(FAILING_REMINDER_HOURS), + }, + }); + }); + + it("announces a recovery once, and says nothing about an ok that was never broken", () => { + const told = { notifiedState: "failing" as const, notifiedAt: T0 }; + expect(decideNotice(told, "ok", at(2)).send).toBe("recovered"); + expect( + decideNotice({ notifiedState: "ok", notifiedAt: at(2) }, "ok", at(3)), + ).toEqual({ + send: null, + next: null, + }); + expect(decideNotice(null, "ok", T0)).toEqual({ send: null, next: null }); + }); + + it("forgets silently when a signal stops applying", () => { + expect( + decideNotice( + { notifiedState: "failing", notifiedAt: T0 }, + "inactive", + at(1), + ), + ).toEqual({ send: null, next: { notifiedState: null, notifiedAt: null } }); + expect(decideNotice(null, "inactive", T0)).toEqual({ + send: null, + next: null, + }); + }); +}); + +function readiness( + states: Partial>, + now: Date, +): RecoveryReadiness { + return { + state: "ok", + evaluatedAt: now.toISOString(), + signals: READINESS_SIGNALS.map((id) => { + const [state, reason] = states[id] ?? ["ok", "fresh"]; + return { + id, + state, + reason, + at: null, + params: { days: 12, accounts: 1 }, + detail: null, + fixPath: READINESS_FIX_PATHS[id], + }; + }), + }; +} + +const sent = () => + dispatch.mock.calls.map(([opts]) => { + const o = opts as unknown as { + userId: string; + titleKey: string; + messageKey: string; + }; + return `${o.userId} ${o.titleKey.split(".").at(-1)} ${o.messageKey}`; + }); + +describe("notifyReadinessChanges", () => { + beforeEach(() => { + store.notices.clear(); + store.admins = [{ id: "admin-a" }, { id: "admin-b" }]; + dispatch.mockClear(); + }); + + it("sends each notice to every admin, through the backup event", async () => { + await notifyReadinessChanges( + readiness({ weekly_copy: ["attention", "stale"] }, T0), + ); + expect(sent()).toEqual([ + "admin-a titleAttention admin.recoveryReadiness.reason.weekly_copy.stale", + "admin-b titleAttention admin.recoveryReadiness.reason.weekly_copy.stale", + ]); + const first = dispatch.mock.calls[0][0] as unknown as { + eventType: string; + params: Record; + metadata: Record; + }; + expect(first.eventType).toBe("BACKUP_HEALTH"); + expect(first.params).toEqual({ days: 12, accounts: 1 }); + expect(first.metadata).toMatchObject({ + url: "/admin/backups", + signal: "weekly_copy", + }); + }); + + it("runs an hourly sequence without nagging: change, silence, escalation, daily reminder, recovery", async () => { + store.admins = [{ id: "admin-a" }]; + const run = (h: number, state: ReadinessState, reason: string) => + notifyReadinessChanges( + readiness({ restore_drill: [state, reason] }, at(h)), + ); + + await run(0, "attention", "failed"); + for (let h = 1; h < 5; h++) await run(h, "attention", "failed"); + expect(sent()).toHaveLength(1); + + await run(5, "failing", "failed"); + for (let h = 6; h < 5 + FAILING_REMINDER_HOURS; h++) + await run(h, "failing", "failed"); + expect(sent()).toHaveLength(2); + + await run(5 + FAILING_REMINDER_HOURS, "failing", "failed"); + await run(6 + FAILING_REMINDER_HOURS, "failing", "failed"); + expect(sent()).toHaveLength(3); + expect(sent()[2]).toContain("titleReminder"); + + await run(40, "ok", "passed"); + await run(41, "ok", "passed"); + expect(sent()).toHaveLength(4); + expect(sent()[3]).toBe( + "admin-a titleRecovered admin.recoveryReadiness.recovered.restore_drill", + ); + }); + + it("sends one notice when two evaluations race on the same row", async () => { + const r = readiness({ recovery_kit: ["attention", "out_of_date"] }, T0); + await Promise.all([notifyReadinessChanges(r), notifyReadinessChanges(r)]); + expect(sent()).toHaveLength(2); // one notice, two admins + }); + + it("claims nothing while there is no admin to tell", async () => { + store.admins = []; + const r = readiness({ offhost_setup: ["attention", "not_configured"] }, T0); + await notifyReadinessChanges(r); + expect(store.notices.get("offhost_setup")?.notifiedState).toBeNull(); + store.admins = [{ id: "admin-a" }]; + await notifyReadinessChanges(r); + expect(sent()).toHaveLength(1); + }); + + it("remembers since when a signal applies, and forgets it when it stops", async () => { + await notifyReadinessChanges( + readiness({ database_dump: ["ok", "fresh"] }, T0), + ); + expect(store.notices.get("database_dump")?.watchedSince).toEqual(T0); + await notifyReadinessChanges( + readiness({ database_dump: ["ok", "fresh"] }, at(5)), + ); + expect(store.notices.get("database_dump")?.watchedSince).toEqual(T0); + await notifyReadinessChanges( + readiness({ database_dump: ["inactive", "disabled"] }, at(6)), + ); + expect(store.notices.get("database_dump")?.watchedSince).toBeNull(); + }); +}); diff --git a/src/lib/recovery/database-dump-health.ts b/src/lib/recovery/database-dump-health.ts new file mode 100644 index 000000000..76bb908c8 --- /dev/null +++ b/src/lib/recovery/database-dump-health.ts @@ -0,0 +1,50 @@ +/** + * The scheduled database dump, as the recovery readiness model sees it. + * + * The dump is the one artifact that rebuilds a lost host: it holds every + * account, every grant and every credential, and a schema change cannot leave + * anything out of it. Its health is a signal of its own in the readiness + * model, read through a provider so the model does not reach into the dump's + * module and a test can hand in any state it wants. + * + * A provider that answers `null` means "this build does not schedule a dump", + * which the model reports as not available rather than as healthy or broken. + */ + +/** + * What the dump module reports about its own schedule. Mirrors + * `DatabaseBackupHealth` in `src/lib/database-backup/health.ts` field for + * field, so the provider below becomes that module's + * `getDatabaseBackupHealth` without a translation. + */ +export interface DatabaseBackupHealth { + /** `DB_BACKUP_ENABLED` is not off. */ + enabled: boolean; + /** The directory dumps are kept in (`DB_BACKUP_DIR`). */ + directory: string; + /** This process can write there. False means dumps only go off-host. */ + localDirWritable: boolean; + /** The off-host bucket is configured, or a dump has been uploaded to it. */ + offhostConfigured: boolean; + /** The newest dump that was kept on the server or reached the bucket. */ + lastGood: { + at: string; + fileName: string; + sizeBytes: number | null; + local: boolean; + offhost: boolean; + } | null; + /** The newest run that failed, or that a dying process left unfinished. */ + lastFailure: { at: string; error: string } | null; +} + +export type DatabaseBackupHealthProvider = ( + now?: Date, +) => Promise; + +/** + * The provider every caller uses unless it passes its own: the dump's health, + * or `null` while no dump is scheduled by this build. + */ +export const readDatabaseBackupHealth: DatabaseBackupHealthProvider = + async () => null; diff --git a/src/lib/recovery/readiness-model.ts b/src/lib/recovery/readiness-model.ts new file mode 100644 index 000000000..962bb05f5 --- /dev/null +++ b/src/lib/recovery/readiness-model.ts @@ -0,0 +1,422 @@ +/** + * Recovery readiness: can this instance be put back after a loss, judged from + * every fact the server holds about its own copies, in one place. + * + * Every one of these facts used to be a pull, on a page nobody opens until the + * day it matters. The weekly copy stopped for 46 days and showed a perfectly + * ordinary row. A failed monthly drill was on the failing-jobs card three days + * in thirty. An off-host bucket that was never configured showed only on the + * backups page, and the key step confirmed one key of the several a restore + * needs. This module turns them into six signals with one verdict each, which + * the readiness tile shows and the hourly evaluation pushes to every admin. + * + * Pure: the inputs are read elsewhere (`readiness.ts`), so each threshold is a + * unit test away and the tile and the push cannot disagree about a verdict. + * + * States: + * ok — nothing to do. A grace period (`*_grace` reasons) is ok too: + * the admin is shown the step, nobody is pushed about it yet. + * attention — something will hurt on the day of a restore; worth a push. + * failing — the instance cannot be restored the way it is meant to be. + * inactive — the signal does not apply here (no dump in this build, no + * off-host bucket for the drill). Never pushed, never counts + * towards the overall verdict. + */ +import { + classifyOffhostBackup, + type OffhostBackupFreshness, +} from "@/lib/jobs/offhost-backup-freshness"; +import type { DatabaseBackupHealth } from "@/lib/recovery/database-dump-health"; +import type { KitEntry } from "@/lib/recovery/kit-rules"; + +const HOUR_MS = 60 * 60 * 1000; +const DAY_MS = 24 * HOUR_MS; + +/** A dump older than this, while dumps are on, means no usable dump. */ +export const DUMP_FAILING_AFTER_HOURS = 36; +/** The weekly copy runs every Sunday; past nine days one Sunday was missed. */ +export const WEEKLY_COPY_ATTENTION_AFTER_DAYS = 9; +/** Past fourteen days two Sundays produced nothing for some account. */ +export const WEEKLY_COPY_FAILING_AFTER_DAYS = 14; +/** + * How long after the first admin exists the off-host bucket and the recovery + * kit may stay unset before the admins are pushed about it. + */ +export const SETUP_GRACE_DAYS = 7; + +export const READINESS_SIGNALS = [ + "database_dump", + "weekly_copy", + "offhost_copies", + "restore_drill", + "offhost_setup", + "recovery_kit", +] as const; +export type ReadinessSignalId = (typeof READINESS_SIGNALS)[number]; + +export const READINESS_STATES = [ + "ok", + "attention", + "failing", + "inactive", +] as const; +export type ReadinessState = (typeof READINESS_STATES)[number]; + +/** Where each signal is fixed. Every path is an admin page. */ +export const READINESS_FIX_PATHS: Record = { + database_dump: "/admin/backups", + weekly_copy: "/admin/backups", + offhost_copies: "/admin/backups", + restore_drill: "/admin/backups", + offhost_setup: "/admin/backups", + recovery_kit: "/admin/encryption", +}; + +export interface ReadinessSignal { + id: ReadinessSignalId; + state: ReadinessState; + /** + * Why, as a closed code per signal. The tile and the push render it through + * `admin.recoveryReadiness..`, so it never carries prose. + */ + reason: string; + /** The instant the signal is about (the newest dump, the last drill). */ + at: string | null; + /** Values the sentence for `reason` interpolates. Never key material. */ + params: Record; + /** + * An operator-facing line from the source (a dump's or a drill's failure + * text), already redacted where it was written. Shown in the tile only. + */ + detail: string | null; + fixPath: string; +} + +export interface RecoveryReadiness { + /** The worst state of the signals that apply; `ok` when none do. */ + state: Exclude; + evaluatedAt: string; + signals: ReadinessSignal[]; +} + +/** What the evaluation reads, gathered by `readiness.ts`. */ +export interface ReadinessInputs { + now: Date; + /** When the first admin account was created; null on an empty install. */ + firstAdminAt: Date | null; + /** + * `null` while no dump is scheduled by this build; `"unreadable"` when the + * dump module threw while reporting, which is itself a failure to report. + */ + dump: DatabaseBackupHealth | null | "unreadable"; + /** + * Since when the evaluation has seen dumps switched on. A dump that has + * never been attempted is judged against this, so an upgrade that turns + * dumps on is not failing before the first run had a chance. + */ + dumpWatchedSince: Date | null; + weekly: { + /** One entry per account: when it was created and its newest copy. */ + accounts: ReadonlyArray<{ createdAt: Date; lastCopyAt: Date | null }>; + }; + offhost: { + configured: boolean; + accounts: ReadonlyArray<{ + lastAttemptAt: Date | null; + lastSuccessAt: Date | null; + }>; + }; + drill: { + lastRunAt: Date; + lastSuccessAt: Date | null; + lastFailureAt: Date | null; + lastFailureReason: string | null; + accountsChecked: number; + accountsFailed: number; + } | null; + kit: { + /** A confirmation exists at all. */ + confirmedAt: Date | null; + /** Keys the kit lists that the confirmation does not cover. */ + unconfirmed: ReadonlyArray>; + }; +} + +function signal( + id: ReadinessSignalId, + state: ReadinessState, + reason: string, + at: Date | null, + params: Record = {}, + detail: string | null = null, +): ReadinessSignal { + return { + id, + state, + reason, + at: at?.toISOString() ?? null, + params, + detail, + fixPath: READINESS_FIX_PATHS[id], + }; +} + +const wholeHours = (ms: number) => Math.max(0, Math.floor(ms / HOUR_MS)); +const wholeDays = (ms: number) => Math.max(0, Math.floor(ms / DAY_MS)); + +function setupGraceEndsAt(firstAdminAt: Date | null): Date | null { + return firstAdminAt + ? new Date(firstAdminAt.getTime() + SETUP_GRACE_DAYS * DAY_MS) + : null; +} + +export function evaluateDatabaseDump( + input: Pick, +): ReadinessSignal { + const { dump, now } = input; + const id = "database_dump"; + if (dump === null) return signal(id, "inactive", "not_available", null); + if (dump === "unreadable") return signal(id, "failing", "unreadable", null); + if (!dump.enabled) return signal(id, "inactive", "disabled", null); + const lastGoodAt = dump.lastGood ? new Date(dump.lastGood.at) : null; + const lastFailureAt = dump.lastFailure ? new Date(dump.lastFailure.at) : null; + const failureReason = dump.lastFailure?.error ?? null; + // A directory this process cannot write is a valid setup when the dumps go + // to the bucket; with no bucket either, no dump can be kept anywhere. + if (!dump.localDirWritable && !dump.offhostConfigured) { + return signal(id, "failing", "nowhere_to_write", lastGoodAt, { + directory: dump.directory, + }); + } + const limitMs = DUMP_FAILING_AFTER_HOURS * HOUR_MS; + const failedSinceGood = + lastFailureAt !== null && + (lastGoodAt === null || lastFailureAt > lastGoodAt); + if (lastGoodAt === null) { + if (failedSinceGood) { + return signal( + id, + "failing", + "never_succeeded", + lastFailureAt, + {}, + failureReason, + ); + } + const watched = input.dumpWatchedSince ?? now; + return now.getTime() - watched.getTime() > limitMs + ? signal(id, "failing", "never_ran", null, { + hours: DUMP_FAILING_AFTER_HOURS, + }) + : signal(id, "ok", "first_pending", null); + } + const ageMs = now.getTime() - lastGoodAt.getTime(); + if (ageMs > limitMs) { + return signal( + id, + "failing", + "stale", + lastGoodAt, + { hours: wholeHours(ageMs) }, + failedSinceGood ? failureReason : null, + ); + } + if (failedSinceGood) { + return signal( + id, + "attention", + "last_failed", + lastGoodAt, + {}, + failureReason, + ); + } + return signal(id, "ok", "fresh", lastGoodAt); +} + +/** + * The worst account decides. An account without any copy ages from its + * creation, so a new account is not overdue before its first Sunday, and one + * the pass keeps skipping becomes overdue like a stale copy would. + */ +export function evaluateWeeklyCopy( + input: Pick, +): ReadinessSignal { + const id = "weekly_copy"; + const { now } = input; + if (input.weekly.accounts.length === 0) { + return signal(id, "inactive", "no_accounts", null); + } + let worstMs = -Infinity; + let worst: { createdAt: Date; lastCopyAt: Date | null } | null = null; + let overdue = 0; + for (const account of input.weekly.accounts) { + const ref = account.lastCopyAt ?? account.createdAt; + const ageMs = now.getTime() - ref.getTime(); + if (ageMs > WEEKLY_COPY_ATTENTION_AFTER_DAYS * DAY_MS) overdue += 1; + if (ageMs > worstMs) { + worstMs = ageMs; + worst = account; + } + } + const days = wholeDays(worstMs); + const at = worst?.lastCopyAt ?? null; + const params = { days, accounts: overdue }; + if (worstMs > WEEKLY_COPY_FAILING_AFTER_DAYS * DAY_MS) { + return signal(id, "failing", at ? "stale" : "never", at, params); + } + if (worstMs > WEEKLY_COPY_ATTENTION_AFTER_DAYS * DAY_MS) { + return signal(id, "attention", at ? "stale" : "never", at, params); + } + // The oldest of the newest copies: the one a restore would be furthest + // behind with. Null only while no account has been copied yet. + const oldestCopy = input.weekly.accounts + .map((a) => a.lastCopyAt) + .filter((d): d is Date => d !== null) + .reduce((min, d) => (min === null || d < min ? d : min), null); + return signal(id, "ok", oldestCopy ? "fresh" : "first_pending", oldestCopy); +} + +/** + * Any account whose newest off-host object is two schedule periods old (the + * freshness module's `stale`, which carries the DST and long-night grace), or + * that a run reached and never put an object there. + */ +export function evaluateOffhostCopies( + input: Pick, +): ReadinessSignal { + const id = "offhost_copies"; + if (!input.offhost.configured) { + return signal(id, "inactive", "not_configured", null); + } + const verdicts: Array<{ + freshness: OffhostBackupFreshness; + lastSuccessAt: Date | null; + }> = input.offhost.accounts.map((a) => ({ + freshness: classifyOffhostBackup({ + lastAttemptAt: a.lastAttemptAt, + lastSuccessAt: a.lastSuccessAt, + now: input.now, + }).freshness, + lastSuccessAt: a.lastSuccessAt, + })); + const oldest = (rows: typeof verdicts) => + rows + .map((r) => r.lastSuccessAt) + .filter((d): d is Date => d !== null) + .reduce( + (min, d) => (min === null || d < min ? d : min), + null, + ); + const behind = verdicts.filter( + (v) => v.freshness === "stale" || v.freshness === "never", + ); + if (behind.length > 0) { + return signal(id, "attention", "stale", oldest(behind), { + accounts: behind.length, + }); + } + const seen = verdicts.filter((v) => v.freshness !== "unknown"); + if (seen.length === 0) return signal(id, "ok", "first_pending", null); + return signal(id, "ok", "fresh", oldest(seen)); +} + +/** A failed drill stays the verdict until a later drill passes. */ +export function evaluateRestoreDrill( + input: Pick, +): ReadinessSignal { + const id = "restore_drill"; + const { drill } = input; + if (!input.offhost.configured) { + return signal(id, "inactive", "not_configured", null); + } + if (drill === null) return signal(id, "ok", "not_run_yet", null); + const failing = + drill.lastFailureAt !== null && + (drill.lastSuccessAt === null || drill.lastFailureAt > drill.lastSuccessAt); + if (failing) { + return signal( + id, + "failing", + // A drill that checked nothing stopped before any object: the bucket + // could not be listed, or held nothing to check. + drill.accountsChecked === 0 ? "failed_run" : "failed", + drill.lastFailureAt, + { failed: drill.accountsFailed, checked: drill.accountsChecked }, + drill.lastFailureReason, + ); + } + return signal(id, "ok", "passed", drill.lastSuccessAt ?? drill.lastRunAt, { + checked: drill.accountsChecked, + }); +} + +export function evaluateOffhostSetup( + input: Pick, +): ReadinessSignal { + const id = "offhost_setup"; + if (input.offhost.configured) return signal(id, "ok", "configured", null); + const graceEnds = setupGraceEndsAt(input.firstAdminAt); + if (graceEnds === null) return signal(id, "inactive", "no_admin", null); + return input.now > graceEnds + ? signal(id, "attention", "not_configured", null) + : signal(id, "ok", "not_configured_grace", graceEnds); +} + +export function evaluateRecoveryKit( + input: Pick, +): ReadinessSignal { + const id = "recovery_kit"; + const { kit } = input; + const keys = kit.unconfirmed.map((k) => k.id).join(", "); + if (kit.unconfirmed.length === 0 && kit.confirmedAt !== null) { + return signal(id, "ok", "confirmed", kit.confirmedAt); + } + if (kit.confirmedAt !== null) { + // Confirmed once, and a key it needs is not in that confirmation: a + // rotation, a new backup key, a re-keyed install. No grace: the keys are + // already in use. + return signal(id, "attention", "out_of_date", kit.confirmedAt, { + keys, + count: kit.unconfirmed.length, + }); + } + const graceEnds = setupGraceEndsAt(input.firstAdminAt); + if (graceEnds === null) return signal(id, "inactive", "no_admin", null); + return input.now > graceEnds + ? signal(id, "attention", "not_confirmed", null, { + keys, + count: kit.unconfirmed.length, + }) + : signal(id, "ok", "not_confirmed_grace", graceEnds, { + keys, + count: kit.unconfirmed.length, + }); +} + +const SEVERITY: Record, number> = { + ok: 0, + attention: 1, + failing: 2, +}; + +export function evaluateRecoveryReadiness( + inputs: ReadinessInputs, +): RecoveryReadiness { + const signals = [ + evaluateDatabaseDump(inputs), + evaluateWeeklyCopy(inputs), + evaluateOffhostCopies(inputs), + evaluateRestoreDrill(inputs), + evaluateOffhostSetup(inputs), + evaluateRecoveryKit(inputs), + ]; + const state = signals.reduce>( + (worst, s) => + s.state !== "inactive" && SEVERITY[s.state] > SEVERITY[worst] + ? s.state + : worst, + "ok", + ); + return { state, evaluatedAt: inputs.now.toISOString(), signals }; +} diff --git a/src/lib/recovery/readiness-notify.ts b/src/lib/recovery/readiness-notify.ts new file mode 100644 index 000000000..4b491c40b --- /dev/null +++ b/src/lib/recovery/readiness-notify.ts @@ -0,0 +1,205 @@ +/** + * Push recovery readiness to every admin: once when a signal turns to + * attention or failing, once when it is back to ok, and a reminder at most + * once a day while it stays failing. Nothing else. + * + * The memory is one row per signal (`RecoveryReadinessNotice`): the state the + * admins were last told about and when. A notice is claimed with a + * compare-and-set on both columns before it is sent, so two workers that + * evaluate in the same minute send it once, and a crash between the claim and + * the send loses a notice rather than repeating it every hour. + * + * Delivery goes through the ordinary localised path, as the deploy-failure + * alert does: every ADMIN, in their own locale, on the channels they enabled + * for `BACKUP_HEALTH`, with the dispatcher's cooldowns and hard-reject rules. + * An admin who switched the event off on a channel is not reached there. + */ +import { prisma } from "@/lib/db"; +import { dispatchLocalisedNotification } from "@/lib/notifications/dispatch-localised"; +import { + READINESS_SIGNALS, + type ReadinessSignal, + type ReadinessSignalId, + type ReadinessState, + type RecoveryReadiness, +} from "@/lib/recovery/readiness-model"; + +/** A failing signal is reminded at most this often. */ +export const FAILING_REMINDER_HOURS = 24; + +type NotifiedState = Exclude; + +export interface StoredNotice { + notifiedState: NotifiedState | null; + notifiedAt: Date | null; +} + +export type NoticeKind = "problem" | "reminder" | "recovered"; + +export interface NoticeDecision { + /** What to send, or null for nothing. */ + send: NoticeKind | null; + /** What to store, or null to leave the row as it is. */ + next: StoredNotice | null; +} + +const NOTHING: NoticeDecision = { send: null, next: null }; + +/** + * Pure: what one evaluation of one signal sends and stores. + * + * - attention / failing, different from what was last announced: send it. + * - failing, as announced, and the last notice is a day old: remind. + * - ok after attention or failing: announce the recovery. + * - inactive: forget silently. A signal that stops applying (the bucket was + * unset, so there is no drill to judge) did not recover, and saying so + * would be untrue; if it applies again and is still broken, it is news. + */ +export function decideNotice( + stored: StoredNotice | null, + current: ReadinessState, + now: Date, +): NoticeDecision { + const last = stored?.notifiedState ?? null; + if (current === "inactive") { + return last === null + ? NOTHING + : { send: null, next: { notifiedState: null, notifiedAt: null } }; + } + if (current === "ok") { + return last === "attention" || last === "failing" + ? { send: "recovered", next: { notifiedState: "ok", notifiedAt: now } } + : NOTHING; + } + if (last !== current) { + return { + send: "problem", + next: { notifiedState: current, notifiedAt: now }, + }; + } + const lastAt = stored?.notifiedAt ?? null; + if ( + current === "failing" && + (lastAt === null || + now.getTime() - lastAt.getTime() >= FAILING_REMINDER_HOURS * 3_600_000) + ) { + return { + send: "reminder", + next: { notifiedState: current, notifiedAt: now }, + }; + } + return NOTHING; +} + +/** The title of a notice, by what it says. */ +function titleKey(kind: NoticeKind, state: ReadinessState): string { + if (kind === "recovered") { + return "notifications.recoveryReadiness.titleRecovered"; + } + if (kind === "reminder") { + return "notifications.recoveryReadiness.titleReminder"; + } + return state === "failing" + ? "notifications.recoveryReadiness.titleFailing" + : "notifications.recoveryReadiness.titleAttention"; +} + +/** + * The body: the same sentence the tile shows for the signal's reason, or the + * signal's own recovery line. + */ +function messageKey(kind: NoticeKind, signal: ReadinessSignal): string { + return kind === "recovered" + ? `admin.recoveryReadiness.recovered.${signal.id}` + : `admin.recoveryReadiness.reason.${signal.id}.${signal.reason}`; +} + +export interface NotifyOutcome { + /** Notices claimed and sent, by signal. */ + sent: Array<{ signal: ReadinessSignalId; kind: NoticeKind }>; + /** Admins each notice went to. */ + admins: number; +} + +function isNotifiedState(value: string | null): value is NotifiedState { + return value === "ok" || value === "attention" || value === "failing"; +} + +/** + * Compare a fresh evaluation against what the admins were last told, claim + * every notice that is due, and send it. + */ +export async function notifyReadinessChanges( + readiness: RecoveryReadiness, + now: Date = new Date(readiness.evaluatedAt), +): Promise { + await prisma.recoveryReadinessNotice.createMany({ + data: READINESS_SIGNALS.map((signal) => ({ signal })), + skipDuplicates: true, + }); + const rows = await prisma.recoveryReadinessNotice.findMany(); + const byId = new Map(rows.map((row) => [row.signal, row])); + const admins = await prisma.user.findMany({ + where: { role: "ADMIN" }, + select: { id: true }, + }); + + const outcome: NotifyOutcome = { sent: [], admins: admins.length }; + for (const signal of readiness.signals) { + const row = byId.get(signal.id); + const applies = signal.state !== "inactive"; + // Since when the signal applies: set once, cleared when it stops. + if (row && applies !== (row.watchedSince !== null)) { + await prisma.recoveryReadinessNotice.updateMany({ + where: { signal: signal.id, watchedSince: row.watchedSince }, + data: { watchedSince: applies ? now : null }, + }); + } + + const stored: StoredNotice | null = row + ? { + notifiedState: isNotifiedState(row.notifiedState) + ? row.notifiedState + : null, + notifiedAt: row.notifiedAt, + } + : null; + const decision = decideNotice(stored, signal.state, now); + // With no admin yet there is nobody to tell, so nothing is claimed: the + // first admin hears about a problem that is still standing. + if (!decision.next || admins.length === 0) continue; + + // Claim: only the evaluation that moves the row from what it read sends. + const claimed = await prisma.recoveryReadinessNotice.updateMany({ + where: { + signal: signal.id, + notifiedState: row?.notifiedState ?? null, + notifiedAt: row?.notifiedAt ?? null, + }, + data: { + notifiedState: decision.next.notifiedState, + notifiedAt: decision.next.notifiedAt, + }, + }); + if (claimed.count !== 1 || decision.send === null) continue; + + for (const admin of admins) { + await dispatchLocalisedNotification({ + userId: admin.id, + eventType: "BACKUP_HEALTH", + titleKey: titleKey(decision.send, signal.state), + messageKey: messageKey(decision.send, signal), + params: signal.params, + metadata: { + source: "recovery-readiness", + url: signal.fixPath, + signal: signal.id, + state: signal.state, + reason: signal.reason, + }, + }); + } + outcome.sent.push({ signal: signal.id, kind: decision.send }); + } + return outcome; +} diff --git a/src/lib/recovery/readiness.ts b/src/lib/recovery/readiness.ts new file mode 100644 index 000000000..3e0c1553d --- /dev/null +++ b/src/lib/recovery/readiness.ts @@ -0,0 +1,117 @@ +/** + * Read every fact the recovery readiness model judges, and judge it. + * + * One reader for both consumers: the admin tile (`GET + * /api/admin/recovery-readiness`) and the hourly evaluation that pushes state + * changes to the admins. Each fact comes from where its own surface already + * reads it, so the tile can never disagree with the backups page about a copy: + * + * - the weekly copy: the `WEEKLY_AUTO` row per account, which never ages out + * of any retention window (pg-boss forgets a run after seven days); + * - the off-host copies: the per-account ledger the nightly run writes, and + * the same "configured" rule the backups page uses (the environment, or a + * ledger row as proof that an upload happened); + * - the drill: its persisted last outcome, with no time window; + * - the kit: the same assessment the kit card shows; + * - the dump: whatever the dump module reports, through a provider. + */ +import { prisma } from "@/lib/db"; +import { offhostBackupConfigured } from "@/lib/jobs/offhost-backup"; +import { + readDatabaseBackupHealth, + type DatabaseBackupHealthProvider, +} from "@/lib/recovery/database-dump-health"; +import { readRecoveryKit } from "@/lib/recovery/kit"; +import { + evaluateRecoveryReadiness, + type ReadinessInputs, + type RecoveryReadiness, +} from "@/lib/recovery/readiness-model"; + +/** The type the weekly worker writes; anything else was uploaded by hand. */ +const SCHEDULED_BACKUP_TYPE = "WEEKLY_AUTO"; + +export interface ReadRecoveryReadinessOptions { + now?: Date; + dumpHealth?: DatabaseBackupHealthProvider; +} + +export async function readReadinessInputs( + options: ReadRecoveryReadinessOptions = {}, +): Promise { + const now = options.now ?? new Date(); + const dumpHealth = options.dumpHealth ?? readDatabaseBackupHealth; + const [firstAdmin, accounts, drill, dumpNotice, dump, kit] = + await Promise.all([ + prisma.user.findFirst({ + where: { role: "ADMIN" }, + orderBy: { createdAt: "asc" }, + select: { createdAt: true }, + }), + prisma.user.findMany({ + select: { + createdAt: true, + dataBackups: { + where: { type: SCHEDULED_BACKUP_TYPE }, + select: { createdAt: true }, + }, + offhostBackupState: { + select: { lastAttemptAt: true, lastSuccessAt: true }, + }, + }, + }), + prisma.restoreDrillOutcome.findUnique({ where: { id: "singleton" } }), + prisma.recoveryReadinessNotice.findUnique({ + where: { signal: "database_dump" }, + select: { watchedSince: true }, + }), + dumpHealth(now).catch(() => "unreadable" as const), + readRecoveryKit(now), + ]); + + // Evidence beats this process's environment, as on the backups page: a + // ledger row is proof an upload happened, whichever process reads it. + const configured = + offhostBackupConfigured() || + accounts.some((account) => account.offhostBackupState !== null); + + return { + now, + firstAdminAt: firstAdmin?.createdAt ?? null, + dump, + dumpWatchedSince: dumpNotice?.watchedSince ?? null, + weekly: { + accounts: accounts.map((account) => ({ + createdAt: account.createdAt, + lastCopyAt: account.dataBackups[0]?.createdAt ?? null, + })), + }, + offhost: { + configured, + accounts: accounts.map((account) => ({ + lastAttemptAt: account.offhostBackupState?.lastAttemptAt ?? null, + lastSuccessAt: account.offhostBackupState?.lastSuccessAt ?? null, + })), + }, + drill: drill + ? { + lastRunAt: drill.lastRunAt, + lastSuccessAt: drill.lastSuccessAt, + lastFailureAt: drill.lastFailureAt, + lastFailureReason: drill.lastFailureReason, + accountsChecked: drill.accountsChecked, + accountsFailed: drill.accountsFailed, + } + : null, + kit: { + confirmedAt: kit.confirmedAt ? new Date(kit.confirmedAt) : null, + unconfirmed: kit.unconfirmed, + }, + }; +} + +export async function readRecoveryReadiness( + options: ReadRecoveryReadinessOptions = {}, +): Promise { + return evaluateRecoveryReadiness(await readReadinessInputs(options)); +} diff --git a/tests/integration/recovery-readiness-notify.test.ts b/tests/integration/recovery-readiness-notify.test.ts new file mode 100644 index 000000000..2f199f482 --- /dev/null +++ b/tests/integration/recovery-readiness-notify.test.ts @@ -0,0 +1,295 @@ +/** + * Recovery readiness, end to end against Postgres: a weekly copy that stopped + * reaches every admin exactly once per state change, through the real + * readiness reader, the real notice ledger, the real dispatcher with its + * preference lookup and ledger, and the real APNs sender (vendor client + * stubbed, the shape `delegated-intake-notification.test.ts` uses). + * + * The facts: + * + * 1. A stale copy becomes one `BACKUP_HEALTH` delivery per admin; the next + * hourly runs at the same state add none. + * 2. Escalating to failing adds one; a reminder comes a day later and not + * sooner; the recovery adds one; ok after that adds none. + * 3. An admin who switched the event off on the channel gets nothing, and + * the notice is still spent, so switching it back on later does not + * replay an old alarm. + * 4. The hourly job runs the same path from its handler. + * + * The ledger write is fire-and-forget in `recordPushAttempt`, so counts are + * polled up to the expected number and then asserted exactly. + */ +import { beforeEach, describe, expect, it, vi } from "vitest"; + +process.env.ENCRYPTION_KEY ??= + "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"; + +const apnsSendMock = vi.fn(); + +vi.mock("@parse/node-apn", () => { + class Provider { + constructor(_opts: unknown) { + void _opts; + } + send(...args: unknown[]) { + return apnsSendMock(...args); + } + shutdown() {} + } + class Notification { + public topic = ""; + public alert: unknown; + public sound: string | undefined; + public badge: number | undefined; + public threadId: string | undefined; + public collapseId: string | undefined; + public payload: unknown; + } + return { default: { Provider, Notification }, Provider, Notification }; +}); + +vi.mock("@/lib/db-compat", () => ({ + ensureDbCompatibility: vi.fn().mockResolvedValue(undefined), +})); + +import { getPrismaClient, truncateAllTables } from "./setup"; + +const ADMIN_ID = "user-readiness-admin"; +const SECOND_ADMIN_ID = "user-readiness-admin-2"; +const MEMBER_ID = "user-readiness-member"; +const DAY = 86_400_000; +const HOUR = 3_600_000; + +const TEST_EC_PEM_LINES = [ + "-----BEGIN PRIVATE KEY-----", + "MIGTAgEAMBMGByqGSM49AgEGCCqGSM49AwEHBHkwdwIBAQQgLOXP3Exjr5L5tamN", + "pTxck85Iaum80PdRlWDpc/ezviOgCgYIKoZIzj0DAQehRANCAAT1x8nKRb8KshQU", + "1aPieSCqOY6ilgC959umaFSlhfav8eZ91UHP/xond9aMoZcuQ7lJG/Rsj70SWMvZ", + "bw81BG89", + "-----END PRIVATE KEY-----", +]; + +const APNS_ENV = { + APNS_KEY_ID: "ABCDE12345", + APNS_TEAM_ID: "TEAM123456", + APNS_BUNDLE_ID: "test.healthlog.ios", + APNS_KEY: TEST_EC_PEM_LINES.join("\\n"), +}; + +async function seedApnsChannel(userId: string): Promise { + const { encrypt } = await import("@/lib/crypto"); + const channel = await getPrismaClient().notificationChannel.create({ + data: { userId, type: "APNS", enabled: true, config: encrypt("{}") }, + }); + await getPrismaClient().device.create({ + data: { + userId, + platform: "ios", + token: `device-${userId}`, + bundleId: APNS_ENV.APNS_BUNDLE_ID, + apnsToken: `apns-${userId}`, + apnsEnvironment: "sandbox", + }, + }); + return channel.id; +} + +async function backupAttempts( + userId: string, + expected: number, +): Promise { + const count = () => + getPrismaClient().pushAttempt.count({ + where: { userId, eventType: "BACKUP_HEALTH" }, + }); + for (let attempt = 0; attempt < 40; attempt += 1) { + if ((await count()) >= expected) break; + await new Promise((r) => setTimeout(r, 25)); + } + // A short settle, so a delivery that should NOT have happened has the same + // chance to land as one that should. + await new Promise((r) => setTimeout(r, 75)); + return count(); +} + +/** Put the member's weekly copy `ageDays` before `now`. */ +async function weeklyCopyAged(ageDays: number, now: Date): Promise { + await getPrismaClient().dataBackup.upsert({ + where: { userId_type: { userId: MEMBER_ID, type: "WEEKLY_AUTO" } }, + create: { + userId: MEMBER_ID, + type: "WEEKLY_AUTO", + data: "sealed", + createdAt: new Date(now.getTime() - ageDays * DAY), + }, + update: { createdAt: new Date(now.getTime() - ageDays * DAY) }, + }); +} + +/** One hourly evaluation at `now`, through the reader and the notifier. */ +async function evaluateAt(now: Date) { + const { readRecoveryReadiness } = await import("@/lib/recovery/readiness"); + const { notifyReadinessChanges } = + await import("@/lib/recovery/readiness-notify"); + const readiness = await readRecoveryReadiness({ now }); + await notifyReadinessChanges(readiness, now); + return readiness; +} + +beforeEach(async () => { + vi.clearAllMocks(); + for (const [k, v] of Object.entries(APNS_ENV)) process.env[k] = v; + delete process.env.APNS_PRODUCTION; + for (const name of [ + "BACKUP_S3_ENDPOINT", + "BACKUP_S3_BUCKET", + "BACKUP_S3_ACCESS_KEY", + "BACKUP_S3_SECRET_KEY", + "BACKUP_ENCRYPTION_KEY", + ]) { + delete process.env[name]; + } + const apns = await import("@/lib/notifications/senders/apns"); + apns.resetApnsForTesting(); + const { __resetDispatchLocaleCacheForTests } = + await import("@/lib/notifications/dispatch-localised"); + __resetDispatchLocaleCacheForTests(); + + const prisma = getPrismaClient(); + await truncateAllTables(prisma); + // Admins created "now": the setup grace keeps the off-host and kit signals + // quiet, so the weekly copy is the only signal that moves. + await prisma.user.createMany({ + data: [ + { + id: ADMIN_ID, + username: "readiness-admin", + email: "readiness-admin@example.test", + role: "ADMIN", + locale: "en", + }, + { + id: SECOND_ADMIN_ID, + username: "readiness-admin-2", + email: "readiness-admin-2@example.test", + role: "ADMIN", + locale: "de", + }, + { + id: MEMBER_ID, + username: "readiness-member", + email: "readiness-member@example.test", + createdAt: new Date(Date.now() - 90 * DAY), + }, + ], + }); + // An admin's own account needs a fresh copy too, or it would age from its + // creation and move the verdict on its own. + const now = new Date(); + for (const userId of [ADMIN_ID, SECOND_ADMIN_ID]) { + await prisma.dataBackup.create({ + data: { userId, type: "WEEKLY_AUTO", data: "sealed", createdAt: now }, + }); + } + + apnsSendMock.mockImplementation(async (note: unknown) => { + void note; + return { sent: [{ device: "x" }], failed: [] }; + }); +}); + +describe("recovery readiness push", () => { + it("notifies every admin once per state change, reminds daily while failing, and announces the recovery once", async () => { + await seedApnsChannel(ADMIN_ID); + await seedApnsChannel(SECOND_ADMIN_ID); + const t0 = new Date(); + const at = (hours: number) => new Date(t0.getTime() + hours * HOUR); + + // Ten days: attention. + await weeklyCopyAged(10, t0); + const first = await evaluateAt(t0); + expect(first.signals.find((s) => s.id === "weekly_copy")).toMatchObject({ + state: "attention", + reason: "stale", + }); + expect(first.signals.find((s) => s.id === "recovery_kit")?.state).toBe( + "ok", + ); + await evaluateAt(at(1)); + await evaluateAt(at(2)); + expect(await backupAttempts(ADMIN_ID, 1)).toBe(1); + expect(await backupAttempts(SECOND_ADMIN_ID, 1)).toBe(1); + + // Each admin in their own language. + const titles = apnsSendMock.mock.calls.map( + ([note]) => (note as { alert: { title: string } }).alert.title, + ); + expect(titles).toContain("Backups need attention"); + expect(titles).toContain("Sicherungen brauchen Aufmerksamkeit"); + + // Fifteen days: failing, once. + await weeklyCopyAged(15, at(3)); + await evaluateAt(at(3)); + await evaluateAt(at(4)); + expect(await backupAttempts(ADMIN_ID, 2)).toBe(2); + + // Still failing: nothing until a day after the last notice, then one. + await evaluateAt(at(3 + 23)); + expect(await backupAttempts(ADMIN_ID, 3)).toBe(2); + await evaluateAt(at(3 + 24)); + await evaluateAt(at(3 + 25)); + expect(await backupAttempts(ADMIN_ID, 3)).toBe(3); + + // A fresh copy: one recovery notice, then silence. + await weeklyCopyAged(0, at(30)); + await evaluateAt(at(30)); + await evaluateAt(at(31)); + expect(await backupAttempts(ADMIN_ID, 4)).toBe(4); + expect(await backupAttempts(SECOND_ADMIN_ID, 4)).toBe(4); + + const notice = await getPrismaClient().recoveryReadinessNotice.findUnique({ + where: { signal: "weekly_copy" }, + }); + expect(notice?.notifiedState).toBe("ok"); + }); + + it("respects an admin's per-channel opt-out and still spends the notice", async () => { + const channelId = await seedApnsChannel(ADMIN_ID); + await getPrismaClient().notificationPreference.create({ + data: { channelId, eventType: "BACKUP_HEALTH", enabled: false }, + }); + const t0 = new Date(); + await weeklyCopyAged(10, t0); + await evaluateAt(t0); + expect(await backupAttempts(ADMIN_ID, 1)).toBe(0); + + await getPrismaClient().notificationPreference.updateMany({ + where: { channelId }, + data: { enabled: true }, + }); + await evaluateAt(new Date(t0.getTime() + HOUR)); + expect(await backupAttempts(ADMIN_ID, 1)).toBe(0); + const notice = await getPrismaClient().recoveryReadinessNotice.findUnique({ + where: { signal: "weekly_copy" }, + }); + expect(notice?.notifiedState).toBe("attention"); + }); + + it("runs the same path from the hourly job", async () => { + await seedApnsChannel(ADMIN_ID); + await weeklyCopyAged(10, new Date()); + const { handleRecoveryReadiness } = + await import("@/lib/jobs/recovery-readiness"); + const outcome = await handleRecoveryReadiness([]); + expect(outcome).toMatchObject({ + ok: true, + did: { + recovery_readiness_state: "attention", + recovery_readiness_notices: 1, + recovery_readiness_admins: 2, + }, + }); + await handleRecoveryReadiness([]); + expect(await backupAttempts(ADMIN_ID, 1)).toBe(1); + }); +});