mirror of
https://github.com/MBombeck/HealthLog.git
synced 2026-11-11 05:42:19 +00:00
feat(admin): judge every backup signal and push changes to admins
Every backup failure was a pull, on a page nobody opens until the day it matters: the weekly copy stopped for 46 days and showed an ordinary row, a failed monthly drill was on the failing-jobs card three days in thirty (job-failures.ts reads 72 hours), and an off-host bucket that was never set up showed only on the backups page. One readiness model (src/lib/recovery/readiness-model.ts) judges six signals as ok, attention or failing, with a reason code, the time it is about and where to fix it: - database dump: failing once the newest good dump is older than 36 h while dumps are on, or when it can be kept nowhere; read through a provider that reports "not available" until the dump lands; - weekly account copies: worst account decides, attention past 9 days, failing past 14; an account without a copy ages from its creation; - off-host copies: attention when any account is two nights behind; - restore drill: failing until a later drill passes, with no time window; the drill now keeps its verdict in restore_drill_outcomes; - off-host setup: attention 7 days after the first admin exists; - recovery kit: attention when a needed key is not confirmed, by id, 7 days after setup or as soon as a confirmed kit is out of date. A "Recovery readiness" card shows them on the admin overview and the backups page. An hourly job (recovery-readiness, :33) sends each change to every admin through the localised admin path, under a new admin-only BACKUP_HEALTH event that the channel preferences route: once when a signal turns to attention or failing, once a day while it stays failing, once when it recovers. Each notice is claimed with a compare-and-set before it is sent, so two workers never send the same one, and nothing is claimed while no admin exists. The backups page calls the weekly copy stale at the same instant as the model (past nine days rather than past ten).
This commit is contained in:
1 parent
d3c672cc3f
commit
7dab9cc5e9
40 files changed
+3386
-37
No files matched your search
+29
-25
@@ -37138,6 +37138,7 @@ components:
|
||||
- SECURITY_ALERT
|
||||
- DAILY_BRIEFING
|
||||
- SHARED_RECORD_INTAKE
|
||||
- BACKUP_HEALTH
|
||||
enabled:
|
||||
type: boolean
|
||||
required:
|
||||
@@ -38519,7 +38520,7 @@ components:
|
||||
documents.sources.tokenRequired, documents.sources.unavailable, documents.sources.unreachable,
|
||||
documents.sources.versionTooOld; encounter — encounter.invalid, encounter.link.invalid,
|
||||
encounter.practitioner-not-found, encounter.reminder-conflict, encounter.reminder-not-found; encryption
|
||||
— encryption.keyBackup.stale, encryption.key_mismatch; environment — environment.backfill_pending,
|
||||
— encryption.key_mismatch; environment — environment.backfill_pending,
|
||||
environment.backfill_rate_limited, environment.invalid, environment.no_home,
|
||||
environment.range_too_large, environment.travel.not_found, environment.travel_suggestion.not_found;
|
||||
export — export.selection.empty, export.selection.unknown_leaf; familyHistory — familyHistory.invalid;
|
||||
@@ -38580,30 +38581,30 @@ components:
|
||||
profile.update.emailInUse, profile.update.emailRateLimited, profile.update.emailSecondFactorRequired,
|
||||
profile.update.invalidBody, profile.update.nothingSaved; profile_claim — profile_claim.invalid,
|
||||
profile_claim.oidc_only_unsupported, profile_claim.sso_unavailable, profile_claim.taken; record_write —
|
||||
record_write.rate_limited; report-selection — report-selection.body.invalid_json,
|
||||
report-selection.body.invalid_shape, report-selection.leaves.unknown; route — route.retired; search —
|
||||
search.invalid; security_activity — security_activity.invalid; share-link —
|
||||
share-link.document.decryptFailed, share-link.document.notFound, share-link.document.rateLimited,
|
||||
share-link.documents.invalid, share-link.report.notFound, share-link.report.rateLimited,
|
||||
share-link.selection.forbidden_leaf, share-link.selection.unknown_leaf; sharing —
|
||||
sharing.accept.expired, sharing.accept.not_found, sharing.accept.not_pending, sharing.accept.revoked,
|
||||
sharing.access.denied, sharing.invite.duplicate, sharing.invite.invalid, sharing.invite.invalid_scope,
|
||||
sharing.invite.manage_browser_only, sharing.invite.no_account, sharing.invite.refused,
|
||||
sharing.invite.self, sharing.not_permitted, sharing.renounce.already_ended, sharing.renounce.not_found,
|
||||
sharing.revoke.already_ended, sharing.revoke.not_found, sharing.session.changed, sharing.switch.invalid,
|
||||
sharing.switch.wrong_transport; source-priority — source-priority.body.invalid_json,
|
||||
source-priority.body.invalid_shape; symptoms — symptoms.definition.limitReached,
|
||||
symptoms.definition.notFound, symptoms.episode.notOpen, symptoms.event.notFound, symptoms.invalid;
|
||||
timeline — timeline.invalid; tokens — tokens.documents.ceiling_reached,
|
||||
tokens.measurements.ceiling_reached, tokens.mirror.not_measurement_token,
|
||||
tokens.workouts.ceiling_reached; vaccination — vaccination.booster-invalid,
|
||||
vaccination.booster-no-antigen, vaccination.booster-out-of-scope, vaccination.custom-vaccine-not-found,
|
||||
vaccination.custom.invalid, vaccination.custom.name-taken, vaccination.custom.not-found,
|
||||
vaccination.encounter-not-found, vaccination.identity-required, vaccination.invalid,
|
||||
vaccination.link.invalid, vaccination.not-found, vaccination.practitioner-not-found; workout —
|
||||
workout.batch.invalid, workout.batch.payload_too_large, workout.batch.source_not_permitted,
|
||||
workout.batch.too_large, workout.delete.synced_source; (unprefixed) — about_me_conflict,
|
||||
ai_response_truncated, coach_prefs_conflict, dashboard_layout_conflict, empty_file,
|
||||
record_write.rate_limited; recovery — recovery.kit.stale; report-selection —
|
||||
report-selection.body.invalid_json, report-selection.body.invalid_shape,
|
||||
report-selection.leaves.unknown; route — route.retired; search — search.invalid; security_activity —
|
||||
security_activity.invalid; share-link — share-link.document.decryptFailed, share-link.document.notFound,
|
||||
share-link.document.rateLimited, share-link.documents.invalid, share-link.report.notFound,
|
||||
share-link.report.rateLimited, share-link.selection.forbidden_leaf, share-link.selection.unknown_leaf;
|
||||
sharing — sharing.accept.expired, sharing.accept.not_found, sharing.accept.not_pending,
|
||||
sharing.accept.revoked, sharing.access.denied, sharing.invite.duplicate, sharing.invite.invalid,
|
||||
sharing.invite.invalid_scope, sharing.invite.manage_browser_only, sharing.invite.no_account,
|
||||
sharing.invite.refused, sharing.invite.self, sharing.not_permitted, sharing.renounce.already_ended,
|
||||
sharing.renounce.not_found, sharing.revoke.already_ended, sharing.revoke.not_found,
|
||||
sharing.session.changed, sharing.switch.invalid, sharing.switch.wrong_transport; source-priority —
|
||||
source-priority.body.invalid_json, source-priority.body.invalid_shape; symptoms —
|
||||
symptoms.definition.limitReached, symptoms.definition.notFound, symptoms.episode.notOpen,
|
||||
symptoms.event.notFound, symptoms.invalid; timeline — timeline.invalid; tokens —
|
||||
tokens.documents.ceiling_reached, tokens.measurements.ceiling_reached,
|
||||
tokens.mirror.not_measurement_token, tokens.workouts.ceiling_reached; vaccination —
|
||||
vaccination.booster-invalid, vaccination.booster-no-antigen, vaccination.booster-out-of-scope,
|
||||
vaccination.custom-vaccine-not-found, vaccination.custom.invalid, vaccination.custom.name-taken,
|
||||
vaccination.custom.not-found, vaccination.encounter-not-found, vaccination.identity-required,
|
||||
vaccination.invalid, vaccination.link.invalid, vaccination.not-found,
|
||||
vaccination.practitioner-not-found; workout — workout.batch.invalid, workout.batch.payload_too_large,
|
||||
workout.batch.source_not_permitted, workout.batch.too_large, workout.delete.synced_source; (unprefixed)
|
||||
— about_me_conflict, ai_response_truncated, coach_prefs_conflict, dashboard_layout_conflict, empty_file,
|
||||
health_score_config_conflict, insights_layout_conflict, invalid_base_updated_at,
|
||||
json_carries_no_intake_time, json_not_an_array, medication_layout_conflict, missing_required_columns,
|
||||
modules_conflict, mood_tag_layout_conflict, not_configured, notification_prefs_conflict, oidc_only,
|
||||
@@ -87952,6 +87953,7 @@ components:
|
||||
- SECURITY_ALERT
|
||||
- DAILY_BRIEFING
|
||||
- SHARED_RECORD_INTAKE
|
||||
- BACKUP_HEALTH
|
||||
description: Notification category a per-channel preference can toggle.
|
||||
additionalProperties:
|
||||
type: boolean
|
||||
@@ -87973,6 +87975,7 @@ components:
|
||||
- SECURITY_ALERT
|
||||
- DAILY_BRIEFING
|
||||
- SHARED_RECORD_INTAKE
|
||||
- BACKUP_HEALTH
|
||||
description: What each event does on a channel with no stored row, resolved for this account by the function delivery
|
||||
uses. Most are constant per category; the mood reminder follows the account's mood-reminder setting
|
||||
(`moodReminderEnabled` on `/api/user/profile`). Render a switch without a row from this map rather than from
|
||||
@@ -88043,6 +88046,7 @@ components:
|
||||
- SECURITY_ALERT
|
||||
- DAILY_BRIEFING
|
||||
- SHARED_RECORD_INTAKE
|
||||
- BACKUP_HEALTH
|
||||
description: Notification category a per-channel preference can toggle.
|
||||
enabled:
|
||||
type: boolean
|
||||
|
||||
@@ -8606,6 +8606,96 @@
|
||||
"confirmedAt": "Bestätigt am {date}.",
|
||||
"confirmedAtBy": "Bestätigt am {date} von {who}.",
|
||||
"dueAgain": "Das Kit wurde am {date} bestätigt, führt jetzt aber Schlüssel auf, die diese Bestätigung nicht abdeckt: {keys}."
|
||||
},
|
||||
"recoveryReadiness": {
|
||||
"title": "Wiederherstellbarkeit",
|
||||
"description": "Ob sich dieser Server nach einem Ausfall wiederherstellen ließe, beurteilt anhand jeder Sicherung, die er anlegt.",
|
||||
"loading": "Sicherungen werden geprüft…",
|
||||
"loadError": "Die Wiederherstellbarkeit konnte nicht geladen werden.",
|
||||
"state": {
|
||||
"ok": "Bereit",
|
||||
"attention": "Braucht Aufmerksamkeit",
|
||||
"failing": "Gefährdet"
|
||||
},
|
||||
"signal": {
|
||||
"database_dump": "Datenbank-Dump",
|
||||
"weekly_copy": "Wöchentliche Kontokopien",
|
||||
"offhost_copies": "Externe Kopien",
|
||||
"restore_drill": "Wiederherstellungsprobe",
|
||||
"offhost_setup": "Externe Sicherung",
|
||||
"recovery_kit": "Wiederherstellungs-Kit"
|
||||
},
|
||||
"at": {
|
||||
"database_dump": "Neuester Dump: {date}",
|
||||
"weekly_copy": "Älteste Kontokopie: {date}",
|
||||
"offhost_copies": "Älteste externe Kopie: {date}",
|
||||
"restore_drill": "Letzte Probe: {date}",
|
||||
"recovery_kit": "Bestätigt: {date}"
|
||||
},
|
||||
"dueBy": "Fällig bis {date}",
|
||||
"fix": {
|
||||
"database_dump": "Sicherungen öffnen",
|
||||
"weekly_copy": "Sicherungen öffnen",
|
||||
"offhost_copies": "Sicherungen öffnen",
|
||||
"restore_drill": "Sicherungen öffnen",
|
||||
"offhost_setup": "Sicherungen öffnen",
|
||||
"recovery_kit": "Kit öffnen"
|
||||
},
|
||||
"reason": {
|
||||
"database_dump": {
|
||||
"not_available": "Diese Version legt keinen geplanten Datenbank-Dump an.",
|
||||
"disabled": "Der geplante Datenbank-Dump ist ausgeschaltet.",
|
||||
"unreadable": "Der Zustand des Datenbank-Dumps konnte nicht gelesen werden.",
|
||||
"never_succeeded": "Noch kein Datenbank-Dump ist gelungen, und der letzte Versuch ist fehlgeschlagen.",
|
||||
"never_ran": "In den {hours} Stunden, seit Dumps eingeschaltet sind, wurde kein Datenbank-Dump geschrieben.",
|
||||
"first_pending": "Der erste Datenbank-Dump ist noch nicht gelaufen.",
|
||||
"stale": "Der neueste Datenbank-Dump ist {hours} Stunden alt.",
|
||||
"last_failed": "Der letzte Datenbank-Dump ist fehlgeschlagen. Der davor ist noch aktuell.",
|
||||
"fresh": "Der neueste Datenbank-Dump ist aktuell.",
|
||||
"nowhere_to_write": "Das Dump-Verzeichnis {directory} ist nicht beschreibbar und es ist kein externer Bucket eingerichtet, daher kann kein Dump aufbewahrt werden."
|
||||
},
|
||||
"weekly_copy": {
|
||||
"no_accounts": "Es gibt noch keine Konten zum Kopieren.",
|
||||
"stale": "Die älteste wöchentliche Kopie ist {days} Tage alt. Konten im Rückstand: {accounts}.",
|
||||
"never": "Ein Konto hat seit {days} Tagen keine wöchentliche Kopie. Konten im Rückstand: {accounts}.",
|
||||
"first_pending": "Die erste wöchentliche Kopie entsteht am frühen Sonntagmorgen.",
|
||||
"fresh": "Jedes Konto hat eine Kopie aus der letzten Woche."
|
||||
},
|
||||
"offhost_copies": {
|
||||
"not_configured": "Es ist kein externer Bucket eingerichtet.",
|
||||
"stale": "Externe Kopien liegen mehr als zwei Nächte zurück. Konten im Rückstand: {accounts}.",
|
||||
"first_pending": "Der nächtliche Lauf für externe Kopien hat noch kein Konto erreicht.",
|
||||
"fresh": "Jedes Konto hat eine aktuelle Kopie im externen Bucket."
|
||||
},
|
||||
"restore_drill": {
|
||||
"not_configured": "Die Probe liest den externen Bucket, und der ist nicht eingerichtet.",
|
||||
"not_run_yet": "Die monatliche Wiederherstellungsprobe ist noch nicht gelaufen.",
|
||||
"failed": "Die letzte Wiederherstellungsprobe ist fehlgeschlagen: {failed} von {checked} Kopien ließen sich nicht zurücklesen.",
|
||||
"failed_run": "Die letzte Wiederherstellungsprobe konnte nicht zu Ende laufen.",
|
||||
"passed": "Die letzte Wiederherstellungsprobe hat {checked} Kopien zurückgelesen."
|
||||
},
|
||||
"offhost_setup": {
|
||||
"configured": "Kopien verlassen diesen Server jede Nacht.",
|
||||
"no_admin": "Es gibt noch kein Admin-Konto.",
|
||||
"not_configured": "Es ist kein externer Bucket eingerichtet, daher liegt jede Kopie nur auf diesem Server.",
|
||||
"not_configured_grace": "Es ist noch kein externer Bucket eingerichtet, daher liegt jede Kopie nur auf diesem Server."
|
||||
},
|
||||
"recovery_kit": {
|
||||
"confirmed": "Von jedem Schlüssel, den eine Wiederherstellung braucht, gibt es eine Kopie.",
|
||||
"out_of_date": "Das Wiederherstellungs-Kit führt Schlüssel auf, für die niemand eine Kopie bestätigt hat: {keys}.",
|
||||
"no_admin": "Es gibt noch kein Admin-Konto.",
|
||||
"not_confirmed": "Niemand hat eine Kopie der Schlüssel bestätigt, die eine Wiederherstellung braucht: {keys}.",
|
||||
"not_confirmed_grace": "Sichere eine Kopie der Schlüssel, die eine Wiederherstellung braucht: {keys}."
|
||||
}
|
||||
},
|
||||
"recovered": {
|
||||
"database_dump": "Der Datenbank-Dump ist wieder aktuell.",
|
||||
"weekly_copy": "Jedes Konto hat wieder eine aktuelle wöchentliche Kopie.",
|
||||
"offhost_copies": "Jedes Konto hat wieder eine aktuelle externe Kopie.",
|
||||
"restore_drill": "Die Wiederherstellungsprobe ist wieder bestanden.",
|
||||
"offhost_setup": "Externe Kopien sind jetzt eingerichtet.",
|
||||
"recovery_kit": "Das Wiederherstellungs-Kit ist wieder bestätigt."
|
||||
}
|
||||
}
|
||||
},
|
||||
"achievements": {
|
||||
@@ -9071,6 +9161,14 @@
|
||||
"checkupReminders": {
|
||||
"title": "Vorsorge-Erinnerungen",
|
||||
"description": "Schalte einzelne Vorsorge-Erinnerungen ein oder aus."
|
||||
},
|
||||
"eventBackupHealth": "Zustand der Sicherungen",
|
||||
"eventBackupHealthDesc": "Wenn eine Sicherung ausbleibt, zurückfällt oder wieder läuft. Nur für Admins.",
|
||||
"recoveryReadiness": {
|
||||
"titleAttention": "Sicherungen brauchen Aufmerksamkeit",
|
||||
"titleFailing": "Sicherungen sind gefährdet",
|
||||
"titleReminder": "Sicherungen sind weiterhin gefährdet",
|
||||
"titleRecovered": "Sicherungen laufen wieder normal"
|
||||
}
|
||||
},
|
||||
"i18n": {
|
||||
|
||||
@@ -8606,6 +8606,96 @@
|
||||
"confirmedAt": "Confirmed on {date}.",
|
||||
"confirmedAtBy": "Confirmed on {date} by {who}.",
|
||||
"dueAgain": "The kit was confirmed on {date}, but it now lists keys that confirmation does not cover: {keys}."
|
||||
},
|
||||
"recoveryReadiness": {
|
||||
"title": "Recovery readiness",
|
||||
"description": "Whether this server could be put back after a loss, judged from every backup it makes.",
|
||||
"loading": "Checking the backups…",
|
||||
"loadError": "Could not load the recovery readiness.",
|
||||
"state": {
|
||||
"ok": "Ready",
|
||||
"attention": "Needs attention",
|
||||
"failing": "At risk"
|
||||
},
|
||||
"signal": {
|
||||
"database_dump": "Database dump",
|
||||
"weekly_copy": "Weekly account copies",
|
||||
"offhost_copies": "Off-host copies",
|
||||
"restore_drill": "Restore drill",
|
||||
"offhost_setup": "Off-host backup",
|
||||
"recovery_kit": "Recovery kit"
|
||||
},
|
||||
"at": {
|
||||
"database_dump": "Newest dump: {date}",
|
||||
"weekly_copy": "Oldest account copy: {date}",
|
||||
"offhost_copies": "Least recent off-host copy: {date}",
|
||||
"restore_drill": "Last drill: {date}",
|
||||
"recovery_kit": "Confirmed: {date}"
|
||||
},
|
||||
"dueBy": "Due by {date}",
|
||||
"fix": {
|
||||
"database_dump": "Open backups",
|
||||
"weekly_copy": "Open backups",
|
||||
"offhost_copies": "Open backups",
|
||||
"restore_drill": "Open backups",
|
||||
"offhost_setup": "Open backups",
|
||||
"recovery_kit": "Open the kit"
|
||||
},
|
||||
"reason": {
|
||||
"database_dump": {
|
||||
"not_available": "This version does not schedule a database dump.",
|
||||
"disabled": "The scheduled database dump is switched off.",
|
||||
"unreadable": "The status of the database dump could not be read.",
|
||||
"never_succeeded": "No database dump has succeeded yet, and the last attempt failed.",
|
||||
"never_ran": "No database dump has been written in the {hours} hours since dumps were switched on.",
|
||||
"first_pending": "The first database dump has not run yet.",
|
||||
"stale": "The newest database dump is {hours} hours old.",
|
||||
"last_failed": "The last database dump failed. The one before it is still recent.",
|
||||
"fresh": "The newest database dump is recent.",
|
||||
"nowhere_to_write": "The dump directory {directory} cannot be written to and no off-host bucket is set up, so no dump can be kept."
|
||||
},
|
||||
"weekly_copy": {
|
||||
"no_accounts": "There are no accounts to copy yet.",
|
||||
"stale": "The oldest weekly copy is {days} days old. Accounts behind: {accounts}.",
|
||||
"never": "An account has had no weekly copy for {days} days. Accounts behind: {accounts}.",
|
||||
"first_pending": "The first weekly copy is made early on Sunday.",
|
||||
"fresh": "Every account has a copy from the last week."
|
||||
},
|
||||
"offhost_copies": {
|
||||
"not_configured": "No off-host bucket is set up.",
|
||||
"stale": "Off-host copies are more than two nights behind. Accounts behind: {accounts}.",
|
||||
"first_pending": "The nightly off-host run has not reached any account yet.",
|
||||
"fresh": "Every account has a recent copy in the off-host bucket."
|
||||
},
|
||||
"restore_drill": {
|
||||
"not_configured": "The drill reads the off-host bucket, which is not set up.",
|
||||
"not_run_yet": "The monthly restore drill has not run yet.",
|
||||
"failed": "The last restore drill failed: {failed} of {checked} copies could not be read back.",
|
||||
"failed_run": "The last restore drill could not finish.",
|
||||
"passed": "The last restore drill read back {checked} copies."
|
||||
},
|
||||
"offhost_setup": {
|
||||
"configured": "Copies leave this server every night.",
|
||||
"no_admin": "There is no admin account yet.",
|
||||
"not_configured": "No off-host bucket is set up, so every copy lives on this server only.",
|
||||
"not_configured_grace": "No off-host bucket is set up yet, so every copy lives on this server only."
|
||||
},
|
||||
"recovery_kit": {
|
||||
"confirmed": "A copy of every key a restore needs is kept.",
|
||||
"out_of_date": "The recovery kit lists keys nobody has confirmed a copy of: {keys}.",
|
||||
"no_admin": "There is no admin account yet.",
|
||||
"not_confirmed": "Nobody has confirmed a copy of the keys a restore needs: {keys}.",
|
||||
"not_confirmed_grace": "Save a copy of the keys a restore needs: {keys}."
|
||||
}
|
||||
},
|
||||
"recovered": {
|
||||
"database_dump": "The database dump is recent again.",
|
||||
"weekly_copy": "Every account has a recent weekly copy again.",
|
||||
"offhost_copies": "Every account has a recent off-host copy again.",
|
||||
"restore_drill": "The restore drill passed again.",
|
||||
"offhost_setup": "Off-host copies are set up now.",
|
||||
"recovery_kit": "The recovery kit is confirmed again."
|
||||
}
|
||||
}
|
||||
},
|
||||
"achievements": {
|
||||
@@ -9071,6 +9161,14 @@
|
||||
"checkupReminders": {
|
||||
"title": "Checkup reminders",
|
||||
"description": "Switch individual checkup reminders on or off."
|
||||
},
|
||||
"eventBackupHealth": "Backup health",
|
||||
"eventBackupHealthDesc": "When a backup stops, falls behind or recovers. Sent to admins only.",
|
||||
"recoveryReadiness": {
|
||||
"titleAttention": "Backups need attention",
|
||||
"titleFailing": "Backups are at risk",
|
||||
"titleReminder": "Backups are still at risk",
|
||||
"titleRecovered": "Backups are back to normal"
|
||||
}
|
||||
},
|
||||
"i18n": {
|
||||
|
||||
@@ -8606,6 +8606,96 @@
|
||||
"confirmedAt": "Confirmado el {date}.",
|
||||
"confirmedAtBy": "Confirmado el {date} por {who}.",
|
||||
"dueAgain": "El kit se confirmó el {date}, pero ahora muestra claves que esa confirmación no cubre: {keys}."
|
||||
},
|
||||
"recoveryReadiness": {
|
||||
"title": "Preparación para la recuperación",
|
||||
"description": "Si este servidor se podría recuperar tras una pérdida, a juzgar por cada copia de seguridad que hace.",
|
||||
"loading": "Comprobando las copias de seguridad…",
|
||||
"loadError": "No se pudo cargar la preparación para la recuperación.",
|
||||
"state": {
|
||||
"ok": "Listo",
|
||||
"attention": "Necesita atención",
|
||||
"failing": "En riesgo"
|
||||
},
|
||||
"signal": {
|
||||
"database_dump": "Volcado de la base de datos",
|
||||
"weekly_copy": "Copias semanales de las cuentas",
|
||||
"offhost_copies": "Copias fuera del host",
|
||||
"restore_drill": "Simulacro de restauración",
|
||||
"offhost_setup": "Copia fuera del host",
|
||||
"recovery_kit": "Kit de recuperación"
|
||||
},
|
||||
"at": {
|
||||
"database_dump": "Volcado más reciente: {date}",
|
||||
"weekly_copy": "Copia de cuenta más antigua: {date}",
|
||||
"offhost_copies": "Copia fuera del host más antigua: {date}",
|
||||
"restore_drill": "Último simulacro: {date}",
|
||||
"recovery_kit": "Confirmado: {date}"
|
||||
},
|
||||
"dueBy": "Vence el {date}",
|
||||
"fix": {
|
||||
"database_dump": "Abrir copias de seguridad",
|
||||
"weekly_copy": "Abrir copias de seguridad",
|
||||
"offhost_copies": "Abrir copias de seguridad",
|
||||
"restore_drill": "Abrir copias de seguridad",
|
||||
"offhost_setup": "Abrir copias de seguridad",
|
||||
"recovery_kit": "Abrir el kit"
|
||||
},
|
||||
"reason": {
|
||||
"database_dump": {
|
||||
"not_available": "Esta versión no programa un volcado de la base de datos.",
|
||||
"disabled": "El volcado programado de la base de datos está desactivado.",
|
||||
"unreadable": "No se pudo leer el estado del volcado de la base de datos.",
|
||||
"never_succeeded": "Ningún volcado de la base de datos ha salido bien todavía, y el último intento falló.",
|
||||
"never_ran": "No se ha escrito ningún volcado de la base de datos en las {hours} horas transcurridas desde que se activaron los volcados.",
|
||||
"first_pending": "El primer volcado de la base de datos aún no se ha ejecutado.",
|
||||
"stale": "El volcado de la base de datos más reciente tiene {hours} horas.",
|
||||
"last_failed": "El último volcado de la base de datos falló. El anterior sigue siendo reciente.",
|
||||
"fresh": "El último volcado de la base de datos es reciente.",
|
||||
"nowhere_to_write": "No se puede escribir en el directorio de volcados {directory} y no hay ningún bucket externo configurado, así que no se puede guardar ningún volcado."
|
||||
},
|
||||
"weekly_copy": {
|
||||
"no_accounts": "Todavía no hay cuentas que copiar.",
|
||||
"stale": "La copia semanal más antigua tiene {days} días. Cuentas con retraso: {accounts}.",
|
||||
"never": "Una cuenta lleva {days} días sin copia semanal. Cuentas con retraso: {accounts}.",
|
||||
"first_pending": "La primera copia semanal se hace el domingo de madrugada.",
|
||||
"fresh": "Cada cuenta tiene una copia de la última semana."
|
||||
},
|
||||
"offhost_copies": {
|
||||
"not_configured": "No hay ningún bucket fuera del host configurado.",
|
||||
"stale": "Las copias fuera del host llevan más de dos noches de retraso. Cuentas con retraso: {accounts}.",
|
||||
"first_pending": "La ejecución nocturna fuera del host aún no ha llegado a ninguna cuenta.",
|
||||
"fresh": "Cada cuenta tiene una copia reciente en el bucket fuera del host."
|
||||
},
|
||||
"restore_drill": {
|
||||
"not_configured": "El simulacro lee el bucket fuera del host, que no está configurado.",
|
||||
"not_run_yet": "El simulacro de restauración mensual aún no se ha ejecutado.",
|
||||
"failed": "El último simulacro de restauración falló: no se pudieron volver a leer {failed} de {checked} copias.",
|
||||
"failed_run": "El último simulacro de restauración no pudo terminar.",
|
||||
"passed": "El último simulacro de restauración volvió a leer {checked} copias."
|
||||
},
|
||||
"offhost_setup": {
|
||||
"configured": "Las copias salen de este servidor cada noche.",
|
||||
"no_admin": "Todavía no hay ninguna cuenta de administrador.",
|
||||
"not_configured": "No hay ningún bucket fuera del host configurado, así que cada copia está solo en este servidor.",
|
||||
"not_configured_grace": "Aún no hay ningún bucket fuera del host configurado, así que cada copia está solo en este servidor."
|
||||
},
|
||||
"recovery_kit": {
|
||||
"confirmed": "Se guarda una copia de cada clave que necesita una restauración.",
|
||||
"out_of_date": "El kit de recuperación muestra claves de las que nadie ha confirmado una copia: {keys}.",
|
||||
"no_admin": "Todavía no hay ninguna cuenta de administrador.",
|
||||
"not_confirmed": "Nadie ha confirmado una copia de las claves que necesita una restauración: {keys}.",
|
||||
"not_confirmed_grace": "Guarda una copia de las claves que necesita una restauración: {keys}."
|
||||
}
|
||||
},
|
||||
"recovered": {
|
||||
"database_dump": "El volcado de la base de datos vuelve a ser reciente.",
|
||||
"weekly_copy": "Cada cuenta vuelve a tener una copia semanal reciente.",
|
||||
"offhost_copies": "Cada cuenta vuelve a tener una copia fuera del host reciente.",
|
||||
"restore_drill": "El simulacro de restauración vuelve a superarse.",
|
||||
"offhost_setup": "Las copias fuera del host ya están configuradas.",
|
||||
"recovery_kit": "El kit de recuperación vuelve a estar confirmado."
|
||||
}
|
||||
}
|
||||
},
|
||||
"achievements": {
|
||||
@@ -9071,6 +9161,14 @@
|
||||
"checkupReminders": {
|
||||
"title": "Recordatorios de chequeos",
|
||||
"description": "Activa o desactiva recordatorios de chequeos concretos."
|
||||
},
|
||||
"eventBackupHealth": "Estado de las copias de seguridad",
|
||||
"eventBackupHealthDesc": "Cuando una copia de seguridad se detiene, se retrasa o se recupera. Solo se envía a los administradores.",
|
||||
"recoveryReadiness": {
|
||||
"titleAttention": "Las copias de seguridad necesitan atención",
|
||||
"titleFailing": "Las copias de seguridad están en riesgo",
|
||||
"titleReminder": "Las copias de seguridad siguen en riesgo",
|
||||
"titleRecovered": "Las copias de seguridad vuelven a la normalidad"
|
||||
}
|
||||
},
|
||||
"i18n": {
|
||||
|
||||
@@ -8606,6 +8606,96 @@
|
||||
"confirmedAt": "Confirmé le {date}.",
|
||||
"confirmedAtBy": "Confirmé le {date} par {who}.",
|
||||
"dueAgain": "Le kit a été confirmé le {date}, mais il liste maintenant des clés que cette confirmation ne couvre pas : {keys}."
|
||||
},
|
||||
"recoveryReadiness": {
|
||||
"title": "Capacité de restauration",
|
||||
"description": "Indique si ce serveur pourrait être remis en route après une perte, d'après chaque sauvegarde qu'il réalise.",
|
||||
"loading": "Vérification des sauvegardes…",
|
||||
"loadError": "Impossible de charger la capacité de restauration.",
|
||||
"state": {
|
||||
"ok": "Prêt",
|
||||
"attention": "Demande de l'attention",
|
||||
"failing": "Menacé"
|
||||
},
|
||||
"signal": {
|
||||
"database_dump": "Dump de la base de données",
|
||||
"weekly_copy": "Copies hebdomadaires des comptes",
|
||||
"offhost_copies": "Copies hors hôte",
|
||||
"restore_drill": "Test de restauration",
|
||||
"offhost_setup": "Sauvegarde hors hôte",
|
||||
"recovery_kit": "Kit de récupération"
|
||||
},
|
||||
"at": {
|
||||
"database_dump": "Dump le plus récent : {date}",
|
||||
"weekly_copy": "Copie de compte la plus ancienne : {date}",
|
||||
"offhost_copies": "Copie hors hôte la plus ancienne : {date}",
|
||||
"restore_drill": "Dernier test : {date}",
|
||||
"recovery_kit": "Confirmé : {date}"
|
||||
},
|
||||
"dueBy": "À faire avant le {date}",
|
||||
"fix": {
|
||||
"database_dump": "Ouvrir les sauvegardes",
|
||||
"weekly_copy": "Ouvrir les sauvegardes",
|
||||
"offhost_copies": "Ouvrir les sauvegardes",
|
||||
"restore_drill": "Ouvrir les sauvegardes",
|
||||
"offhost_setup": "Ouvrir les sauvegardes",
|
||||
"recovery_kit": "Ouvrir le kit"
|
||||
},
|
||||
"reason": {
|
||||
"database_dump": {
|
||||
"not_available": "Cette version ne planifie pas de dump de la base de données.",
|
||||
"disabled": "Le dump planifié de la base de données est désactivé.",
|
||||
"unreadable": "L'état du dump de la base de données n'a pas pu être lu.",
|
||||
"never_succeeded": "Aucun dump de la base de données n'a encore réussi, et la dernière tentative a échoué.",
|
||||
"never_ran": "Aucun dump de la base de données n'a été écrit dans les {hours} heures écoulées depuis l'activation des dumps.",
|
||||
"first_pending": "Le premier dump de la base de données n'a pas encore été lancé.",
|
||||
"stale": "Le dump de la base de données le plus récent date de {hours} heures.",
|
||||
"last_failed": "Le dernier dump de la base de données a échoué. Le précédent est encore récent.",
|
||||
"fresh": "Le dernier dump de la base de données est récent.",
|
||||
"nowhere_to_write": "Le dossier des dumps {directory} n'est pas accessible en écriture et aucun bucket externe n'est configuré, donc aucun dump ne peut être conservé."
|
||||
},
|
||||
"weekly_copy": {
|
||||
"no_accounts": "Il n'y a pas encore de compte à copier.",
|
||||
"stale": "La copie hebdomadaire la plus ancienne date de {days} jours. Comptes en retard : {accounts}.",
|
||||
"never": "Un compte n'a pas eu de copie hebdomadaire depuis {days} jours. Comptes en retard : {accounts}.",
|
||||
"first_pending": "La première copie hebdomadaire est faite tôt le dimanche matin.",
|
||||
"fresh": "Chaque compte a une copie de la semaine dernière."
|
||||
},
|
||||
"offhost_copies": {
|
||||
"not_configured": "Aucun bucket hors hôte n'est configuré.",
|
||||
"stale": "Les copies hors hôte ont plus de deux nuits de retard. Comptes en retard : {accounts}.",
|
||||
"first_pending": "L'envoi nocturne hors hôte n'a encore atteint aucun compte.",
|
||||
"fresh": "Chaque compte a une copie récente dans le bucket hors hôte."
|
||||
},
|
||||
"restore_drill": {
|
||||
"not_configured": "Le test lit le bucket hors hôte, qui n'est pas configuré.",
|
||||
"not_run_yet": "Le test de restauration mensuel n'a pas encore été lancé.",
|
||||
"failed": "Le dernier test de restauration a échoué : {failed} copies sur {checked} n'ont pas pu être relues.",
|
||||
"failed_run": "Le dernier test de restauration n'a pas pu aller jusqu'au bout.",
|
||||
"passed": "Le dernier test de restauration a relu {checked} copies."
|
||||
},
|
||||
"offhost_setup": {
|
||||
"configured": "Les copies quittent ce serveur chaque nuit.",
|
||||
"no_admin": "Il n'y a pas encore de compte administrateur.",
|
||||
"not_configured": "Aucun bucket hors hôte n'est configuré, donc chaque copie se trouve uniquement sur ce serveur.",
|
||||
"not_configured_grace": "Aucun bucket hors hôte n'est encore configuré, donc chaque copie se trouve uniquement sur ce serveur."
|
||||
},
|
||||
"recovery_kit": {
|
||||
"confirmed": "Une copie de chaque clé nécessaire à une restauration est conservée.",
|
||||
"out_of_date": "Le kit de récupération liste des clés dont personne n'a confirmé de copie : {keys}.",
|
||||
"no_admin": "Il n'y a pas encore de compte administrateur.",
|
||||
"not_confirmed": "Personne n'a confirmé de copie des clés nécessaires à une restauration : {keys}.",
|
||||
"not_confirmed_grace": "Sauvegarde une copie des clés nécessaires à une restauration : {keys}."
|
||||
}
|
||||
},
|
||||
"recovered": {
|
||||
"database_dump": "Le dump de la base de données est de nouveau récent.",
|
||||
"weekly_copy": "Chaque compte a de nouveau une copie hebdomadaire récente.",
|
||||
"offhost_copies": "Chaque compte a de nouveau une copie hors hôte récente.",
|
||||
"restore_drill": "Le test de restauration réussit de nouveau.",
|
||||
"offhost_setup": "Les copies hors hôte sont maintenant configurées.",
|
||||
"recovery_kit": "Le kit de récupération est de nouveau confirmé."
|
||||
}
|
||||
}
|
||||
},
|
||||
"achievements": {
|
||||
@@ -9071,6 +9161,14 @@
|
||||
"checkupReminders": {
|
||||
"title": "Rappels de bilans",
|
||||
"description": "Activez ou désactivez chaque rappel de bilan."
|
||||
},
|
||||
"eventBackupHealth": "État des sauvegardes",
|
||||
"eventBackupHealthDesc": "Quand une sauvegarde s'arrête, prend du retard ou repart. Envoyé uniquement aux administrateurs.",
|
||||
"recoveryReadiness": {
|
||||
"titleAttention": "Les sauvegardes demandent de l'attention",
|
||||
"titleFailing": "Les sauvegardes sont menacées",
|
||||
"titleReminder": "Les sauvegardes sont toujours menacées",
|
||||
"titleRecovered": "Les sauvegardes sont revenues à la normale"
|
||||
}
|
||||
},
|
||||
"i18n": {
|
||||
|
||||
@@ -8606,6 +8606,96 @@
|
||||
"confirmedAt": "Confermato il {date}.",
|
||||
"confirmedAtBy": "Confermato il {date} da {who}.",
|
||||
"dueAgain": "Il kit è stato confermato il {date}, ma ora elenca chiavi che quella conferma non copre: {keys}."
|
||||
},
|
||||
"recoveryReadiness": {
|
||||
"title": "Preparazione al ripristino",
|
||||
"description": "Se questo server potrebbe essere rimesso in piedi dopo una perdita, valutato in base a ogni backup che esegue.",
|
||||
"loading": "Controllo dei backup…",
|
||||
"loadError": "Impossibile caricare la preparazione al ripristino.",
|
||||
"state": {
|
||||
"ok": "Pronto",
|
||||
"attention": "Richiede attenzione",
|
||||
"failing": "A rischio"
|
||||
},
|
||||
"signal": {
|
||||
"database_dump": "Dump del database",
|
||||
"weekly_copy": "Copie settimanali degli account",
|
||||
"offhost_copies": "Copie fuori host",
|
||||
"restore_drill": "Prova di ripristino",
|
||||
"offhost_setup": "Backup fuori host",
|
||||
"recovery_kit": "Kit di recupero"
|
||||
},
|
||||
"at": {
|
||||
"database_dump": "Dump più recente: {date}",
|
||||
"weekly_copy": "Copia di account più vecchia: {date}",
|
||||
"offhost_copies": "Copia fuori host meno recente: {date}",
|
||||
"restore_drill": "Ultima prova: {date}",
|
||||
"recovery_kit": "Confermato: {date}"
|
||||
},
|
||||
"dueBy": "Scadenza: {date}",
|
||||
"fix": {
|
||||
"database_dump": "Apri i backup",
|
||||
"weekly_copy": "Apri i backup",
|
||||
"offhost_copies": "Apri i backup",
|
||||
"restore_drill": "Apri i backup",
|
||||
"offhost_setup": "Apri i backup",
|
||||
"recovery_kit": "Apri il kit"
|
||||
},
|
||||
"reason": {
|
||||
"database_dump": {
|
||||
"not_available": "Questa versione non pianifica un dump del database.",
|
||||
"disabled": "Il dump pianificato del database è disattivato.",
|
||||
"unreadable": "Non è stato possibile leggere lo stato del dump del database.",
|
||||
"never_succeeded": "Nessun dump del database è ancora riuscito, e l'ultimo tentativo non è andato a buon fine.",
|
||||
"never_ran": "Nelle {hours} ore trascorse da quando i dump sono stati attivati non è stato scritto alcun dump del database.",
|
||||
"first_pending": "Il primo dump del database non è ancora stato eseguito.",
|
||||
"stale": "Il dump del database più recente ha {hours} ore.",
|
||||
"last_failed": "L'ultimo dump del database non è riuscito. Quello precedente è ancora recente.",
|
||||
"fresh": "L'ultimo dump del database è recente.",
|
||||
"nowhere_to_write": "La cartella dei dump {directory} non è scrivibile e non è configurato nessun bucket esterno, quindi non si può conservare alcun dump."
|
||||
},
|
||||
"weekly_copy": {
|
||||
"no_accounts": "Non ci sono ancora account da copiare.",
|
||||
"stale": "La copia settimanale più vecchia ha {days} giorni. Account in ritardo: {accounts}.",
|
||||
"never": "Un account non ha una copia settimanale da {days} giorni. Account in ritardo: {accounts}.",
|
||||
"first_pending": "La prima copia settimanale viene creata domenica mattina presto.",
|
||||
"fresh": "Ogni account ha una copia dell'ultima settimana."
|
||||
},
|
||||
"offhost_copies": {
|
||||
"not_configured": "Non è configurato alcun bucket fuori host.",
|
||||
"stale": "Le copie fuori host sono indietro di più di due notti. Account in ritardo: {accounts}.",
|
||||
"first_pending": "Il giro notturno fuori host non ha ancora raggiunto alcun account.",
|
||||
"fresh": "Ogni account ha una copia recente nel bucket fuori host."
|
||||
},
|
||||
"restore_drill": {
|
||||
"not_configured": "La prova legge il bucket fuori host, che non è configurato.",
|
||||
"not_run_yet": "La prova di ripristino mensile non è ancora stata eseguita.",
|
||||
"failed": "L'ultima prova di ripristino non è riuscita: {failed} copie su {checked} non si sono potute rileggere.",
|
||||
"failed_run": "L'ultima prova di ripristino non è riuscita ad arrivare in fondo.",
|
||||
"passed": "L'ultima prova di ripristino ha riletto {checked} copie."
|
||||
},
|
||||
"offhost_setup": {
|
||||
"configured": "Le copie lasciano questo server ogni notte.",
|
||||
"no_admin": "Non esiste ancora un account amministratore.",
|
||||
"not_configured": "Non è configurato alcun bucket fuori host, quindi ogni copia si trova solo su questo server.",
|
||||
"not_configured_grace": "Non è ancora configurato alcun bucket fuori host, quindi ogni copia si trova solo su questo server."
|
||||
},
|
||||
"recovery_kit": {
|
||||
"confirmed": "Esiste una copia di ogni chiave necessaria per un ripristino.",
|
||||
"out_of_date": "Il kit di recupero elenca chiavi di cui nessuno ha confermato una copia: {keys}.",
|
||||
"no_admin": "Non esiste ancora un account amministratore.",
|
||||
"not_confirmed": "Nessuno ha confermato una copia delle chiavi necessarie per un ripristino: {keys}.",
|
||||
"not_confirmed_grace": "Salva una copia delle chiavi necessarie per un ripristino: {keys}."
|
||||
}
|
||||
},
|
||||
"recovered": {
|
||||
"database_dump": "Il dump del database è di nuovo recente.",
|
||||
"weekly_copy": "Ogni account ha di nuovo una copia settimanale recente.",
|
||||
"offhost_copies": "Ogni account ha di nuovo una copia fuori host recente.",
|
||||
"restore_drill": "La prova di ripristino è di nuovo riuscita.",
|
||||
"offhost_setup": "Le copie fuori host ora sono configurate.",
|
||||
"recovery_kit": "Il kit di recupero è di nuovo confermato."
|
||||
}
|
||||
}
|
||||
},
|
||||
"achievements": {
|
||||
@@ -9071,6 +9161,14 @@
|
||||
"checkupReminders": {
|
||||
"title": "Promemoria dei controlli",
|
||||
"description": "Attiva o disattiva i singoli promemoria dei controlli."
|
||||
},
|
||||
"eventBackupHealth": "Stato dei backup",
|
||||
"eventBackupHealthDesc": "Quando un backup si ferma, resta indietro o torna a funzionare. Inviato solo agli amministratori.",
|
||||
"recoveryReadiness": {
|
||||
"titleAttention": "I backup richiedono attenzione",
|
||||
"titleFailing": "I backup sono a rischio",
|
||||
"titleReminder": "I backup sono ancora a rischio",
|
||||
"titleRecovered": "I backup sono tornati alla normalità"
|
||||
}
|
||||
},
|
||||
"i18n": {
|
||||
|
||||
@@ -8606,6 +8606,96 @@
|
||||
"confirmedAt": "{date}에 확인됨.",
|
||||
"confirmedAtBy": "{date}에 {who}님이 확인함.",
|
||||
"dueAgain": "키트는 {date}에 확인되었지만 지금은 그 확인에 포함되지 않은 키가 있습니다: {keys}."
|
||||
},
|
||||
"recoveryReadiness": {
|
||||
"title": "복구 준비 상태",
|
||||
"description": "이 서버가 만드는 모든 백업을 기준으로, 서버를 잃었을 때 되살릴 수 있는지 판단합니다.",
|
||||
"loading": "백업을 확인하는 중…",
|
||||
"loadError": "복구 준비 상태를 불러오지 못했습니다.",
|
||||
"state": {
|
||||
"ok": "준비됨",
|
||||
"attention": "확인 필요",
|
||||
"failing": "위험"
|
||||
},
|
||||
"signal": {
|
||||
"database_dump": "데이터베이스 덤프",
|
||||
"weekly_copy": "주간 계정 사본",
|
||||
"offhost_copies": "외부 보관 사본",
|
||||
"restore_drill": "복원 훈련",
|
||||
"offhost_setup": "외부 보관 백업",
|
||||
"recovery_kit": "복구 키트"
|
||||
},
|
||||
"at": {
|
||||
"database_dump": "최신 덤프: {date}",
|
||||
"weekly_copy": "가장 오래된 계정 사본: {date}",
|
||||
"offhost_copies": "가장 오래된 외부 보관 사본: {date}",
|
||||
"restore_drill": "마지막 훈련: {date}",
|
||||
"recovery_kit": "확인: {date}"
|
||||
},
|
||||
"dueBy": "{date}까지",
|
||||
"fix": {
|
||||
"database_dump": "백업 열기",
|
||||
"weekly_copy": "백업 열기",
|
||||
"offhost_copies": "백업 열기",
|
||||
"restore_drill": "백업 열기",
|
||||
"offhost_setup": "백업 열기",
|
||||
"recovery_kit": "키트 열기"
|
||||
},
|
||||
"reason": {
|
||||
"database_dump": {
|
||||
"not_available": "이 버전은 데이터베이스 덤프를 예약하지 않습니다.",
|
||||
"disabled": "예약된 데이터베이스 덤프가 꺼져 있습니다.",
|
||||
"unreadable": "데이터베이스 덤프 상태를 읽지 못했습니다.",
|
||||
"never_succeeded": "아직 성공한 데이터베이스 덤프가 없으며 마지막 시도도 실패했습니다.",
|
||||
"never_ran": "덤프를 켠 뒤 {hours}시간 동안 데이터베이스 덤프가 기록되지 않았습니다.",
|
||||
"first_pending": "첫 데이터베이스 덤프가 아직 실행되지 않았습니다.",
|
||||
"stale": "최신 데이터베이스 덤프가 {hours}시간 전 것입니다.",
|
||||
"last_failed": "마지막 데이터베이스 덤프가 실패했습니다. 그 전 덤프는 아직 최근 것입니다.",
|
||||
"fresh": "최신 데이터베이스 덤프가 최근 것입니다.",
|
||||
"nowhere_to_write": "덤프 디렉터리 {directory}에 쓸 수 없고 외부 버킷도 설정되어 있지 않아 덤프를 보관할 수 없습니다."
|
||||
},
|
||||
"weekly_copy": {
|
||||
"no_accounts": "아직 복사할 계정이 없습니다.",
|
||||
"stale": "가장 오래된 주간 사본이 {days}일 전 것입니다. 뒤처진 계정: {accounts}.",
|
||||
"never": "{days}일 동안 주간 사본이 없는 계정이 있습니다. 뒤처진 계정: {accounts}.",
|
||||
"first_pending": "첫 주간 사본은 일요일 이른 시간에 만들어집니다.",
|
||||
"fresh": "모든 계정에 지난주 사본이 있습니다."
|
||||
},
|
||||
"offhost_copies": {
|
||||
"not_configured": "외부 보관 버킷이 설정되지 않았습니다.",
|
||||
"stale": "외부 보관 사본이 이틀 밤 넘게 뒤처져 있습니다. 뒤처진 계정: {accounts}.",
|
||||
"first_pending": "야간 외부 보관 실행이 아직 어떤 계정에도 도달하지 않았습니다.",
|
||||
"fresh": "모든 계정의 최근 사본이 외부 보관 버킷에 있습니다."
|
||||
},
|
||||
"restore_drill": {
|
||||
"not_configured": "훈련은 외부 보관 버킷을 읽는데, 버킷이 설정되지 않았습니다.",
|
||||
"not_run_yet": "월간 복원 훈련이 아직 실행되지 않았습니다.",
|
||||
"failed": "마지막 복원 훈련이 실패했습니다: 사본 {checked}개 중 {failed}개를 다시 읽지 못했습니다.",
|
||||
"failed_run": "마지막 복원 훈련을 끝까지 마치지 못했습니다.",
|
||||
"passed": "마지막 복원 훈련에서 사본 {checked}개를 다시 읽었습니다."
|
||||
},
|
||||
"offhost_setup": {
|
||||
"configured": "사본이 매일 밤 이 서버 밖으로 나갑니다.",
|
||||
"no_admin": "아직 관리자 계정이 없습니다.",
|
||||
"not_configured": "외부 보관 버킷이 설정되지 않아 모든 사본이 이 서버에만 있습니다.",
|
||||
"not_configured_grace": "외부 보관 버킷이 아직 설정되지 않아 모든 사본이 이 서버에만 있습니다."
|
||||
},
|
||||
"recovery_kit": {
|
||||
"confirmed": "복원에 필요한 모든 키의 사본이 보관되어 있습니다.",
|
||||
"out_of_date": "복구 키트에 아무도 사본을 확인하지 않은 키가 있습니다: {keys}.",
|
||||
"no_admin": "아직 관리자 계정이 없습니다.",
|
||||
"not_confirmed": "복원에 필요한 키의 사본을 아무도 확인하지 않았습니다: {keys}.",
|
||||
"not_confirmed_grace": "복원에 필요한 키의 사본을 보관하세요: {keys}."
|
||||
}
|
||||
},
|
||||
"recovered": {
|
||||
"database_dump": "데이터베이스 덤프가 다시 최신 상태입니다.",
|
||||
"weekly_copy": "모든 계정에 다시 최근 주간 사본이 있습니다.",
|
||||
"offhost_copies": "모든 계정에 다시 최근 외부 보관 사본이 있습니다.",
|
||||
"restore_drill": "복원 훈련을 다시 통과했습니다.",
|
||||
"offhost_setup": "이제 외부 보관 사본이 설정되었습니다.",
|
||||
"recovery_kit": "복구 키트가 다시 확인되었습니다."
|
||||
}
|
||||
}
|
||||
},
|
||||
"achievements": {
|
||||
@@ -9071,6 +9161,14 @@
|
||||
"checkupReminders": {
|
||||
"title": "검진 리마인더",
|
||||
"description": "검진 리마인더를 개별로 켜거나 끕니다."
|
||||
},
|
||||
"eventBackupHealth": "백업 상태",
|
||||
"eventBackupHealthDesc": "백업이 멈추거나 뒤처지거나 다시 정상으로 돌아오면 보내요. 관리자에게만 보내요.",
|
||||
"recoveryReadiness": {
|
||||
"titleAttention": "백업에 확인이 필요해요",
|
||||
"titleFailing": "백업이 위험해요",
|
||||
"titleReminder": "백업이 여전히 위험해요",
|
||||
"titleRecovered": "백업이 다시 정상이에요"
|
||||
}
|
||||
},
|
||||
"i18n": {
|
||||
|
||||
@@ -8606,6 +8606,96 @@
|
||||
"confirmedAt": "Bevestigd op {date}.",
|
||||
"confirmedAtBy": "Bevestigd op {date} door {who}.",
|
||||
"dueAgain": "De kit is bevestigd op {date}, maar bevat nu sleutels die niet onder die bevestiging vallen: {keys}."
|
||||
},
|
||||
"recoveryReadiness": {
|
||||
"title": "Herstelbaarheid",
|
||||
"description": "Of deze server na verlies weer op te bouwen is, beoordeeld op basis van elke back-up die hij maakt.",
|
||||
"loading": "Back-ups controleren…",
|
||||
"loadError": "De herstelbaarheid kon niet worden geladen.",
|
||||
"state": {
|
||||
"ok": "Gereed",
|
||||
"attention": "Aandacht nodig",
|
||||
"failing": "In gevaar"
|
||||
},
|
||||
"signal": {
|
||||
"database_dump": "Databasedump",
|
||||
"weekly_copy": "Wekelijkse accountkopieën",
|
||||
"offhost_copies": "Externe kopieën",
|
||||
"restore_drill": "Hersteltest",
|
||||
"offhost_setup": "Externe back-up",
|
||||
"recovery_kit": "Herstelkit"
|
||||
},
|
||||
"at": {
|
||||
"database_dump": "Nieuwste dump: {date}",
|
||||
"weekly_copy": "Oudste accountkopie: {date}",
|
||||
"offhost_copies": "Oudste externe kopie: {date}",
|
||||
"restore_drill": "Laatste test: {date}",
|
||||
"recovery_kit": "Bevestigd: {date}"
|
||||
},
|
||||
"dueBy": "Uiterlijk {date}",
|
||||
"fix": {
|
||||
"database_dump": "Back-ups openen",
|
||||
"weekly_copy": "Back-ups openen",
|
||||
"offhost_copies": "Back-ups openen",
|
||||
"restore_drill": "Back-ups openen",
|
||||
"offhost_setup": "Back-ups openen",
|
||||
"recovery_kit": "Kit openen"
|
||||
},
|
||||
"reason": {
|
||||
"database_dump": {
|
||||
"not_available": "Deze versie plant geen databasedump in.",
|
||||
"disabled": "De geplande databasedump is uitgeschakeld.",
|
||||
"unreadable": "De status van de databasedump kon niet worden gelezen.",
|
||||
"never_succeeded": "Er is nog geen databasedump gelukt, en de laatste poging is mislukt.",
|
||||
"never_ran": "In de {hours} uur sinds dumps zijn ingeschakeld, is er geen databasedump geschreven.",
|
||||
"first_pending": "De eerste databasedump is nog niet uitgevoerd.",
|
||||
"stale": "De nieuwste databasedump is {hours} uur oud.",
|
||||
"last_failed": "De laatste databasedump is mislukt. De dump daarvoor is nog recent.",
|
||||
"fresh": "De nieuwste databasedump is recent.",
|
||||
"nowhere_to_write": "De dumpmap {directory} is niet beschrijfbaar en er is geen externe bucket ingesteld, dus er kan geen dump worden bewaard."
|
||||
},
|
||||
"weekly_copy": {
|
||||
"no_accounts": "Er zijn nog geen accounts om te kopiëren.",
|
||||
"stale": "De oudste wekelijkse kopie is {days} dagen oud. Achterlopende accounts: {accounts}.",
|
||||
"never": "Een account heeft al {days} dagen geen wekelijkse kopie. Achterlopende accounts: {accounts}.",
|
||||
"first_pending": "De eerste wekelijkse kopie wordt vroeg op zondag gemaakt.",
|
||||
"fresh": "Elk account heeft een kopie van de afgelopen week."
|
||||
},
|
||||
"offhost_copies": {
|
||||
"not_configured": "Er is geen externe bucket ingesteld.",
|
||||
"stale": "Externe kopieën lopen meer dan twee nachten achter. Achterlopende accounts: {accounts}.",
|
||||
"first_pending": "De nachtelijke run voor externe kopieën heeft nog geen enkel account bereikt.",
|
||||
"fresh": "Elk account heeft een recente kopie in de externe bucket."
|
||||
},
|
||||
"restore_drill": {
|
||||
"not_configured": "De test leest de externe bucket, en die is niet ingesteld.",
|
||||
"not_run_yet": "De maandelijkse hersteltest is nog niet uitgevoerd.",
|
||||
"failed": "De laatste hersteltest is mislukt: {failed} van {checked} kopieën konden niet worden teruggelezen.",
|
||||
"failed_run": "De laatste hersteltest kon niet worden afgerond.",
|
||||
"passed": "De laatste hersteltest heeft {checked} kopieën teruggelezen."
|
||||
},
|
||||
"offhost_setup": {
|
||||
"configured": "Kopieën verlaten deze server elke nacht.",
|
||||
"no_admin": "Er is nog geen beheerdersaccount.",
|
||||
"not_configured": "Er is geen externe bucket ingesteld, dus elke kopie staat alleen op deze server.",
|
||||
"not_configured_grace": "Er is nog geen externe bucket ingesteld, dus elke kopie staat alleen op deze server."
|
||||
},
|
||||
"recovery_kit": {
|
||||
"confirmed": "Van elke sleutel die een herstel nodig heeft, wordt een kopie bewaard.",
|
||||
"out_of_date": "De herstelkit bevat sleutels waarvan niemand een kopie heeft bevestigd: {keys}.",
|
||||
"no_admin": "Er is nog geen beheerdersaccount.",
|
||||
"not_confirmed": "Niemand heeft een kopie bevestigd van de sleutels die een herstel nodig heeft: {keys}.",
|
||||
"not_confirmed_grace": "Bewaar een kopie van de sleutels die een herstel nodig heeft: {keys}."
|
||||
}
|
||||
},
|
||||
"recovered": {
|
||||
"database_dump": "De databasedump is weer recent.",
|
||||
"weekly_copy": "Elk account heeft weer een recente wekelijkse kopie.",
|
||||
"offhost_copies": "Elk account heeft weer een recente externe kopie.",
|
||||
"restore_drill": "De hersteltest is weer geslaagd.",
|
||||
"offhost_setup": "Externe kopieën zijn nu ingesteld.",
|
||||
"recovery_kit": "De herstelkit is weer bevestigd."
|
||||
}
|
||||
}
|
||||
},
|
||||
"achievements": {
|
||||
@@ -9071,6 +9161,14 @@
|
||||
"checkupReminders": {
|
||||
"title": "Herinneringen voor controles",
|
||||
"description": "Zet afzonderlijke herinneringen voor controles aan of uit."
|
||||
},
|
||||
"eventBackupHealth": "Status van back-ups",
|
||||
"eventBackupHealthDesc": "Wordt verstuurd als een back-up stopt, achterloopt of weer bijgewerkt is. Alleen voor beheerders.",
|
||||
"recoveryReadiness": {
|
||||
"titleAttention": "Back-ups hebben aandacht nodig",
|
||||
"titleFailing": "Back-ups lopen gevaar",
|
||||
"titleReminder": "Back-ups lopen nog steeds gevaar",
|
||||
"titleRecovered": "Back-ups zijn weer in orde"
|
||||
}
|
||||
},
|
||||
"i18n": {
|
||||
|
||||
@@ -8606,6 +8606,96 @@
|
||||
"confirmedAt": "Potwierdzono {date}.",
|
||||
"confirmedAtBy": "Potwierdzono {date}, potwierdził(a): {who}.",
|
||||
"dueAgain": "Zestaw potwierdzono {date}, ale teraz wymienia klucze, których to potwierdzenie nie obejmuje: {keys}."
|
||||
},
|
||||
"recoveryReadiness": {
|
||||
"title": "Gotowość do odzyskiwania",
|
||||
"description": "Czy ten serwer dałoby się odtworzyć po utracie, oceniane na podstawie każdej kopii zapasowej, którą tworzy.",
|
||||
"loading": "Sprawdzanie kopii zapasowych…",
|
||||
"loadError": "Nie udało się wczytać gotowości do odzyskiwania.",
|
||||
"state": {
|
||||
"ok": "Gotowe",
|
||||
"attention": "Wymaga uwagi",
|
||||
"failing": "Zagrożone"
|
||||
},
|
||||
"signal": {
|
||||
"database_dump": "Zrzut bazy danych",
|
||||
"weekly_copy": "Cotygodniowe kopie kont",
|
||||
"offhost_copies": "Kopie poza hostem",
|
||||
"restore_drill": "Próba przywracania",
|
||||
"offhost_setup": "Kopia poza hostem",
|
||||
"recovery_kit": "Zestaw odzyskiwania"
|
||||
},
|
||||
"at": {
|
||||
"database_dump": "Najnowszy zrzut: {date}",
|
||||
"weekly_copy": "Najstarsza kopia konta: {date}",
|
||||
"offhost_copies": "Najstarsza kopia poza hostem: {date}",
|
||||
"restore_drill": "Ostatnia próba: {date}",
|
||||
"recovery_kit": "Potwierdzono: {date}"
|
||||
},
|
||||
"dueBy": "Termin: {date}",
|
||||
"fix": {
|
||||
"database_dump": "Otwórz kopie zapasowe",
|
||||
"weekly_copy": "Otwórz kopie zapasowe",
|
||||
"offhost_copies": "Otwórz kopie zapasowe",
|
||||
"restore_drill": "Otwórz kopie zapasowe",
|
||||
"offhost_setup": "Otwórz kopie zapasowe",
|
||||
"recovery_kit": "Otwórz zestaw"
|
||||
},
|
||||
"reason": {
|
||||
"database_dump": {
|
||||
"not_available": "Ta wersja nie planuje zrzutu bazy danych.",
|
||||
"disabled": "Zaplanowany zrzut bazy danych jest wyłączony.",
|
||||
"unreadable": "Nie udało się odczytać stanu zrzutu bazy danych.",
|
||||
"never_succeeded": "Żaden zrzut bazy danych jeszcze się nie udał, a ostatnia próba zakończyła się błędem.",
|
||||
"never_ran": "Od włączenia zrzutów minęło {hours} h, a nie zapisano żadnego zrzutu bazy danych.",
|
||||
"first_pending": "Pierwszy zrzut bazy danych jeszcze się nie odbył.",
|
||||
"stale": "Najnowszy zrzut bazy danych ma {hours} h.",
|
||||
"last_failed": "Ostatni zrzut bazy danych nie powiódł się. Poprzedni jest jeszcze aktualny.",
|
||||
"fresh": "Najnowszy zrzut bazy danych jest aktualny.",
|
||||
"nowhere_to_write": "Katalog zrzutów {directory} nie jest zapisywalny i nie skonfigurowano zewnętrznego bucketu, więc żaden zrzut nie może zostać zachowany."
|
||||
},
|
||||
"weekly_copy": {
|
||||
"no_accounts": "Nie ma jeszcze kont do skopiowania.",
|
||||
"stale": "Najstarsza cotygodniowa kopia ma {days} dni. Zaległe konta: {accounts}.",
|
||||
"never": "Jedno z kont od {days} dni nie ma cotygodniowej kopii. Zaległe konta: {accounts}.",
|
||||
"first_pending": "Pierwsza cotygodniowa kopia powstaje wcześnie rano w niedzielę.",
|
||||
"fresh": "Każde konto ma kopię z ostatniego tygodnia."
|
||||
},
|
||||
"offhost_copies": {
|
||||
"not_configured": "Nie skonfigurowano bucketu poza hostem.",
|
||||
"stale": "Kopie poza hostem są opóźnione o ponad dwie noce. Zaległe konta: {accounts}.",
|
||||
"first_pending": "Nocny przebieg kopii poza hostem nie objął jeszcze żadnego konta.",
|
||||
"fresh": "Każde konto ma aktualną kopię w buckecie poza hostem."
|
||||
},
|
||||
"restore_drill": {
|
||||
"not_configured": "Próba odczytuje bucket poza hostem, który nie jest skonfigurowany.",
|
||||
"not_run_yet": "Comiesięczna próba przywracania jeszcze się nie odbyła.",
|
||||
"failed": "Ostatnia próba przywracania nie powiodła się: nie udało się odczytać {failed} z {checked} kopii.",
|
||||
"failed_run": "Ostatnia próba przywracania nie dobiegła końca.",
|
||||
"passed": "Ostatnia próba przywracania odczytała kopie: {checked}."
|
||||
},
|
||||
"offhost_setup": {
|
||||
"configured": "Kopie opuszczają ten serwer każdej nocy.",
|
||||
"no_admin": "Nie ma jeszcze konta administratora.",
|
||||
"not_configured": "Nie skonfigurowano bucketu poza hostem, więc każda kopia jest tylko na tym serwerze.",
|
||||
"not_configured_grace": "Nie skonfigurowano jeszcze bucketu poza hostem, więc każda kopia jest tylko na tym serwerze."
|
||||
},
|
||||
"recovery_kit": {
|
||||
"confirmed": "Istnieje kopia każdego klucza potrzebnego do przywrócenia.",
|
||||
"out_of_date": "Zestaw odzyskiwania wymienia klucze, których kopii nikt nie potwierdził: {keys}.",
|
||||
"no_admin": "Nie ma jeszcze konta administratora.",
|
||||
"not_confirmed": "Nikt nie potwierdził kopii kluczy potrzebnych do przywrócenia: {keys}.",
|
||||
"not_confirmed_grace": "Zapisz kopię kluczy potrzebnych do przywrócenia: {keys}."
|
||||
}
|
||||
},
|
||||
"recovered": {
|
||||
"database_dump": "Zrzut bazy danych znów jest aktualny.",
|
||||
"weekly_copy": "Każde konto znów ma aktualną cotygodniową kopię.",
|
||||
"offhost_copies": "Każde konto znów ma aktualną kopię poza hostem.",
|
||||
"restore_drill": "Próba przywracania znów się powiodła.",
|
||||
"offhost_setup": "Kopie poza hostem są już skonfigurowane.",
|
||||
"recovery_kit": "Zestaw odzyskiwania jest znów potwierdzony."
|
||||
}
|
||||
}
|
||||
},
|
||||
"achievements": {
|
||||
@@ -9071,6 +9161,14 @@
|
||||
"checkupReminders": {
|
||||
"title": "Przypomnienia o badaniach kontrolnych",
|
||||
"description": "Włączaj lub wyłączaj pojedyncze przypomnienia o badaniach."
|
||||
},
|
||||
"eventBackupHealth": "Stan kopii zapasowych",
|
||||
"eventBackupHealthDesc": "Gdy kopia zapasowa przestaje powstawać, zostaje w tyle lub wraca do normy. Wysyłane tylko do administratorów.",
|
||||
"recoveryReadiness": {
|
||||
"titleAttention": "Kopie zapasowe wymagają uwagi",
|
||||
"titleFailing": "Kopie zapasowe są zagrożone",
|
||||
"titleReminder": "Kopie zapasowe są nadal zagrożone",
|
||||
"titleRecovered": "Kopie zapasowe znów działają normalnie"
|
||||
}
|
||||
},
|
||||
"i18n": {
|
||||
|
||||
@@ -8606,6 +8606,96 @@
|
||||
"confirmedAt": "Confirmado em {date}.",
|
||||
"confirmedAtBy": "Confirmado em {date} por {who}.",
|
||||
"dueAgain": "O kit foi confirmado em {date}, mas agora lista chaves que essa confirmação não cobre: {keys}."
|
||||
},
|
||||
"recoveryReadiness": {
|
||||
"title": "Prontidão para recuperação",
|
||||
"description": "Se este servidor poderia ser reconstruído após uma perda, avaliado a partir de cada backup que ele faz.",
|
||||
"loading": "Verificando os backups…",
|
||||
"loadError": "Não foi possível carregar a prontidão para recuperação.",
|
||||
"state": {
|
||||
"ok": "Pronto",
|
||||
"attention": "Precisa de atenção",
|
||||
"failing": "Em risco"
|
||||
},
|
||||
"signal": {
|
||||
"database_dump": "Dump do banco de dados",
|
||||
"weekly_copy": "Cópias semanais das contas",
|
||||
"offhost_copies": "Cópias fora do host",
|
||||
"restore_drill": "Teste de restauração",
|
||||
"offhost_setup": "Backup fora do host",
|
||||
"recovery_kit": "Kit de recuperação"
|
||||
},
|
||||
"at": {
|
||||
"database_dump": "Dump mais recente: {date}",
|
||||
"weekly_copy": "Cópia de conta mais antiga: {date}",
|
||||
"offhost_copies": "Cópia fora do host mais antiga: {date}",
|
||||
"restore_drill": "Último teste: {date}",
|
||||
"recovery_kit": "Confirmado: {date}"
|
||||
},
|
||||
"dueBy": "Prazo: {date}",
|
||||
"fix": {
|
||||
"database_dump": "Abrir backups",
|
||||
"weekly_copy": "Abrir backups",
|
||||
"offhost_copies": "Abrir backups",
|
||||
"restore_drill": "Abrir backups",
|
||||
"offhost_setup": "Abrir backups",
|
||||
"recovery_kit": "Abrir o kit"
|
||||
},
|
||||
"reason": {
|
||||
"database_dump": {
|
||||
"not_available": "Esta versão não agenda um dump do banco de dados.",
|
||||
"disabled": "O dump agendado do banco de dados está desativado.",
|
||||
"unreadable": "Não foi possível ler o status do dump do banco de dados.",
|
||||
"never_succeeded": "Nenhum dump do banco de dados foi concluído com sucesso ainda, e a última tentativa falhou.",
|
||||
"never_ran": "Nenhum dump do banco de dados foi gravado nas {hours} horas desde que os dumps foram ativados.",
|
||||
"first_pending": "O primeiro dump do banco de dados ainda não foi executado.",
|
||||
"stale": "O dump do banco de dados mais recente tem {hours} horas.",
|
||||
"last_failed": "O último dump do banco de dados falhou. O anterior ainda é recente.",
|
||||
"fresh": "O dump do banco de dados mais recente está em dia.",
|
||||
"nowhere_to_write": "Não é possível gravar no diretório de dumps {directory} e nenhum bucket externo está configurado, então nenhum dump pode ser guardado."
|
||||
},
|
||||
"weekly_copy": {
|
||||
"no_accounts": "Ainda não há contas para copiar.",
|
||||
"stale": "A cópia semanal mais antiga tem {days} dias. Contas atrasadas: {accounts}.",
|
||||
"never": "Uma conta está sem cópia semanal há {days} dias. Contas atrasadas: {accounts}.",
|
||||
"first_pending": "A primeira cópia semanal é feita na madrugada de domingo.",
|
||||
"fresh": "Todas as contas têm uma cópia da última semana."
|
||||
},
|
||||
"offhost_copies": {
|
||||
"not_configured": "Nenhum bucket fora do host está configurado.",
|
||||
"stale": "As cópias fora do host estão mais de duas noites atrasadas. Contas atrasadas: {accounts}.",
|
||||
"first_pending": "A execução noturna fora do host ainda não chegou a nenhuma conta.",
|
||||
"fresh": "Todas as contas têm uma cópia recente no bucket fora do host."
|
||||
},
|
||||
"restore_drill": {
|
||||
"not_configured": "O teste lê o bucket fora do host, que não está configurado.",
|
||||
"not_run_yet": "O teste mensal de restauração ainda não foi executado.",
|
||||
"failed": "O último teste de restauração falhou: {failed} de {checked} cópias não puderam ser lidas de volta.",
|
||||
"failed_run": "O último teste de restauração não pôde ser concluído.",
|
||||
"passed": "O último teste de restauração leu de volta {checked} cópias."
|
||||
},
|
||||
"offhost_setup": {
|
||||
"configured": "As cópias saem deste servidor todas as noites.",
|
||||
"no_admin": "Ainda não há uma conta de administrador.",
|
||||
"not_configured": "Nenhum bucket fora do host está configurado, então todas as cópias ficam apenas neste servidor.",
|
||||
"not_configured_grace": "Nenhum bucket fora do host foi configurado ainda, então todas as cópias ficam apenas neste servidor."
|
||||
},
|
||||
"recovery_kit": {
|
||||
"confirmed": "Há uma cópia guardada de cada chave de que uma restauração precisa.",
|
||||
"out_of_date": "O kit de recuperação lista chaves das quais ninguém confirmou uma cópia: {keys}.",
|
||||
"no_admin": "Ainda não há uma conta de administrador.",
|
||||
"not_confirmed": "Ninguém confirmou uma cópia das chaves de que uma restauração precisa: {keys}.",
|
||||
"not_confirmed_grace": "Guarde uma cópia das chaves de que uma restauração precisa: {keys}."
|
||||
}
|
||||
},
|
||||
"recovered": {
|
||||
"database_dump": "O dump do banco de dados está em dia novamente.",
|
||||
"weekly_copy": "Todas as contas têm novamente uma cópia semanal recente.",
|
||||
"offhost_copies": "Todas as contas têm novamente uma cópia recente fora do host.",
|
||||
"restore_drill": "O teste de restauração passou novamente.",
|
||||
"offhost_setup": "As cópias fora do host estão configuradas agora.",
|
||||
"recovery_kit": "O kit de recuperação está confirmado novamente."
|
||||
}
|
||||
}
|
||||
},
|
||||
"achievements": {
|
||||
@@ -9071,6 +9161,14 @@
|
||||
"checkupReminders": {
|
||||
"title": "Lembretes de check-up",
|
||||
"description": "Ligue ou desligue lembretes de check-up individuais."
|
||||
},
|
||||
"eventBackupHealth": "Saúde dos backups",
|
||||
"eventBackupHealthDesc": "Enviado quando um backup para, fica atrasado ou se recupera. Apenas para administradores.",
|
||||
"recoveryReadiness": {
|
||||
"titleAttention": "Os backups precisam de atenção",
|
||||
"titleFailing": "Os backups estão em risco",
|
||||
"titleReminder": "Os backups continuam em risco",
|
||||
"titleRecovered": "Os backups voltaram ao normal"
|
||||
}
|
||||
},
|
||||
"i18n": {
|
||||
|
||||
@@ -135,6 +135,13 @@ const PINNED_AFFORDANCES: Record<
|
||||
// off-host copy is, and the state it paints green came out of the database
|
||||
// rather than out of a request this card just made.
|
||||
"src/components/admin/offhost-backups-section.tsx": { "text-success": 1 },
|
||||
// Reads the server's verdict, writes nothing: the check and the green mark
|
||||
// say a backup signal is ok as the readiness model judged it, not that a
|
||||
// request this card made succeeded.
|
||||
"src/components/admin/recovery-readiness-card.tsx": {
|
||||
CheckCircle2: 2,
|
||||
"text-success": 1,
|
||||
},
|
||||
"src/components/admin/recent-audit-preview.tsx": {
|
||||
CheckCircle2: 2,
|
||||
"text-success": 1,
|
||||
|
||||
@@ -28,6 +28,7 @@ import { ApiTokenOverviewSection } from "@/components/admin/api-token-overview-s
|
||||
import { AppLogPreviewSection } from "@/components/admin/app-log-preview-section";
|
||||
import { BackupsSection } from "@/components/admin/backups-section";
|
||||
import { OffhostBackupsSection } from "@/components/admin/offhost-backups-section";
|
||||
import { RecoveryReadinessCard } from "@/components/admin/recovery-readiness-card";
|
||||
import { DangerZoneSection } from "@/components/admin/danger-zone-section";
|
||||
import { EncryptionSection } from "@/components/admin/encryption-section";
|
||||
import { GeneralSettingsSection } from "@/components/admin/general-settings-section";
|
||||
@@ -166,10 +167,12 @@ export function AdminSectionRenderer({
|
||||
</SectionFrame>
|
||||
);
|
||||
case "backups":
|
||||
// Two cards, two independent failure modes: the copies held in this
|
||||
// database, then the copies that reached the operator's bucket.
|
||||
// The verdict across every backup first, then two cards, two
|
||||
// independent failure modes: the copies held in this database, then
|
||||
// the copies that reached the operator's bucket.
|
||||
return (
|
||||
<SectionFrame>
|
||||
<RecoveryReadinessCard />
|
||||
<BackupsSection />
|
||||
<OffhostBackupsSection />
|
||||
</SectionFrame>
|
||||
|
||||
@@ -4,6 +4,7 @@ import { useAuth } from "@/hooks/use-auth";
|
||||
import { useTranslations } from "@/lib/i18n/context";
|
||||
import { QueryErrorRow } from "@/components/ui/query-error-row";
|
||||
import { AdminShell } from "@/components/admin/admin-shell";
|
||||
import { RecoveryReadinessCard } from "@/components/admin/recovery-readiness-card";
|
||||
import { SystemStatusSummary } from "@/components/admin/system-status-summary";
|
||||
import { VersionTileSection } from "@/components/admin/version-tile-section";
|
||||
import { RecentAuditPreview } from "@/components/admin/recent-audit-preview";
|
||||
@@ -49,6 +50,9 @@ export default function AdminOverviewPage() {
|
||||
|
||||
<SystemStatusSummary />
|
||||
|
||||
{/* v1.46 — every backup signal, the recovery kit included. */}
|
||||
<RecoveryReadinessCard />
|
||||
|
||||
<RecentAuditPreview />
|
||||
</div>
|
||||
</AdminShell>
|
||||
|
||||
@@ -0,0 +1,24 @@
|
||||
/**
|
||||
* GET /api/admin/recovery-readiness
|
||||
*
|
||||
* Every backup signal with its verdict, its time and where to fix it: the
|
||||
* database dump, the weekly copy, the off-host copies, the last restore drill
|
||||
* (no time window: a failed drill stays failed until a later one passes), the
|
||||
* off-host setup and the recovery kit. The same judgement the hourly
|
||||
* evaluation pushes to the admins (`src/lib/recovery/readiness-model.ts`).
|
||||
* Cookie-only admin.
|
||||
*/
|
||||
import { apiHandler, requireAdmin } from "@/lib/api-handler";
|
||||
import { apiSuccess } from "@/lib/api-response";
|
||||
import { annotate } from "@/lib/logging/context";
|
||||
import { readRecoveryReadiness } from "@/lib/recovery/readiness";
|
||||
|
||||
export const dynamic = "force-dynamic";
|
||||
|
||||
export const GET = apiHandler(async () => {
|
||||
await requireAdmin();
|
||||
annotate({ action: { name: "admin.recoveryReadiness.status" } });
|
||||
const readiness = await readRecoveryReadiness();
|
||||
annotate({ meta: { recovery_readiness_state: readiness.state } });
|
||||
return apiSuccess(readiness);
|
||||
});
|
||||
@@ -9,7 +9,10 @@ import {
|
||||
sanitiseZodIssues,
|
||||
} from "@/lib/api-response";
|
||||
import { notificationPreferenceSchema } from "@/lib/validations/notifications";
|
||||
import { EVENT_TYPES, CHANNEL_TYPE_LABELS } from "@/lib/notifications/types";
|
||||
import {
|
||||
CHANNEL_TYPE_LABELS,
|
||||
eventTypesForRole,
|
||||
} from "@/lib/notifications/types";
|
||||
import type { ChannelType } from "@/lib/notifications/types";
|
||||
import { decrypt, encrypt } from "@/lib/crypto";
|
||||
import { resolveEventDefaults } from "@/lib/notifications/event-defaults";
|
||||
@@ -92,7 +95,9 @@ export const GET = apiHandler(async () => {
|
||||
return apiSuccess({
|
||||
channels: channelData,
|
||||
preferences,
|
||||
eventTypes: EVENT_TYPES as unknown as string[],
|
||||
// Admin-only events (backup health) are listed for admins only: nobody
|
||||
// else is ever sent one, so a switch for it would switch nothing.
|
||||
eventTypes: eventTypesForRole(user.role),
|
||||
// What an event does on a channel with no stored row, resolved for this
|
||||
// account by the same function the dispatcher asks. A client renders a
|
||||
// switch without a row from this map, never from its own copy of the
|
||||
|
||||
@@ -53,7 +53,7 @@ vi.mock("@tanstack/react-query", () => ({
|
||||
schedule: {
|
||||
lastSuccessAt: "2026-09-01T03:00:00.000Z",
|
||||
lastSuccessAgeDays: 2,
|
||||
staleAfterDays: 10,
|
||||
staleAfterDays: 9,
|
||||
stale: false,
|
||||
lastRun: null,
|
||||
lastRunFailed: false,
|
||||
|
||||
@@ -6,7 +6,7 @@ import en from "../../../../messages/en.json";
|
||||
const healthySchedule = {
|
||||
lastSuccessAt: "2026-09-01T03:00:00.000Z",
|
||||
lastSuccessAgeDays: 2,
|
||||
staleAfterDays: 10,
|
||||
staleAfterDays: 9,
|
||||
stale: false,
|
||||
lastRun: {
|
||||
at: "2026-09-01T03:00:12.000Z",
|
||||
|
||||
@@ -0,0 +1,154 @@
|
||||
/**
|
||||
* The recovery readiness card.
|
||||
*
|
||||
* 1. every signal renders as its own row with the server's state, its
|
||||
* sentence and its time;
|
||||
* 2. a row that needs work links to where it is fixed, unless that is the
|
||||
* page already open; an ok or inactive row carries no link;
|
||||
* 3. a grace period shows its due date rather than a time of an event;
|
||||
* 4. a failed drill shows its reason with no time window: a failure months
|
||||
* old is still on the card;
|
||||
* 5. the overall badge carries the worst state; the copy resolves in German.
|
||||
*
|
||||
* Mutation check: show the fix link on every row and case 2 goes red; drop
|
||||
* the `detail` line and case 4 goes red.
|
||||
*/
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { renderToStaticMarkup } from "react-dom/server";
|
||||
import { QueryClient, QueryClientProvider } from "@tanstack/react-query";
|
||||
|
||||
import { I18nProvider } from "@/lib/i18n/context";
|
||||
import { queryKeys } from "@/lib/query-keys";
|
||||
import { RecoveryReadinessCard } from "@/components/admin/recovery-readiness-card";
|
||||
import type {
|
||||
ReadinessSignal,
|
||||
RecoveryReadiness,
|
||||
} from "@/components/admin/use-recovery-readiness";
|
||||
import { READINESS_FIX_PATHS } from "@/lib/recovery/readiness-model";
|
||||
import { primeLocales } from "@/__tests__/helpers/prime-locales";
|
||||
|
||||
primeLocales("de");
|
||||
|
||||
const pathname = vi.hoisted(() => ({ current: "/admin" }));
|
||||
vi.mock("next/navigation", () => ({
|
||||
useRouter: () => ({ push: vi.fn(), replace: vi.fn() }),
|
||||
usePathname: () => pathname.current,
|
||||
}));
|
||||
|
||||
function sig(
|
||||
id: ReadinessSignal["id"],
|
||||
state: ReadinessSignal["state"],
|
||||
reason: string,
|
||||
over: Partial<ReadinessSignal> = {},
|
||||
): ReadinessSignal {
|
||||
return {
|
||||
id,
|
||||
state,
|
||||
reason,
|
||||
at: null,
|
||||
params: {},
|
||||
detail: null,
|
||||
fixPath: READINESS_FIX_PATHS[id],
|
||||
...over,
|
||||
};
|
||||
}
|
||||
|
||||
const READINESS: RecoveryReadiness = {
|
||||
state: "failing",
|
||||
evaluatedAt: "2026-10-25T12:00:00.000Z",
|
||||
signals: [
|
||||
sig("database_dump", "inactive", "not_available"),
|
||||
sig("weekly_copy", "attention", "stale", {
|
||||
at: "2026-10-14T03:00:00.000Z",
|
||||
params: { days: 11, accounts: 2 },
|
||||
}),
|
||||
sig("offhost_copies", "ok", "fresh", { at: "2026-10-25T02:40:00.000Z" }),
|
||||
sig("restore_drill", "failing", "failed", {
|
||||
at: "2026-07-01T04:20:00.000Z",
|
||||
params: { failed: 1, checked: 4 },
|
||||
detail: "1 of 4 backup objects failed. Restore drill: bad tag",
|
||||
}),
|
||||
sig("offhost_setup", "ok", "configured"),
|
||||
sig("recovery_kit", "ok", "not_confirmed_grace", {
|
||||
at: "2026-10-30T09:00:00.000Z",
|
||||
params: { keys: "v2, API_TOKEN_HMAC_KEY", count: 2 },
|
||||
}),
|
||||
],
|
||||
};
|
||||
|
||||
function render(readiness: RecoveryReadiness, locale: "en" | "de" = "en") {
|
||||
const client = new QueryClient();
|
||||
client.setQueryData(queryKeys.adminRecoveryReadiness(), readiness);
|
||||
return renderToStaticMarkup(
|
||||
<QueryClientProvider client={client}>
|
||||
<I18nProvider initialLocale={locale}>
|
||||
<RecoveryReadinessCard />
|
||||
</I18nProvider>
|
||||
</QueryClientProvider>,
|
||||
);
|
||||
}
|
||||
|
||||
/** The markup of one signal's row. */
|
||||
function row(html: string, id: string): string {
|
||||
const start = html.indexOf(`data-testid="recovery-signal-${id}"`);
|
||||
expect(start).toBeGreaterThan(-1);
|
||||
const end = html.indexOf("</li>", start);
|
||||
return html.slice(start, end);
|
||||
}
|
||||
|
||||
describe("RecoveryReadinessCard", () => {
|
||||
it("renders every signal with its state and sentence", () => {
|
||||
pathname.current = "/admin";
|
||||
const html = render(READINESS);
|
||||
for (const s of READINESS.signals) {
|
||||
expect(row(html, s.id)).toContain(`data-state="${s.state}"`);
|
||||
}
|
||||
expect(row(html, "weekly_copy")).toContain(
|
||||
"The oldest weekly copy is 11 days old. Accounts behind: 2.",
|
||||
);
|
||||
expect(row(html, "database_dump")).toContain(
|
||||
"This version does not schedule a database dump.",
|
||||
);
|
||||
});
|
||||
|
||||
it("links a row that needs work to where it is fixed, and nothing else", () => {
|
||||
pathname.current = "/admin";
|
||||
const html = render(READINESS);
|
||||
expect(row(html, "weekly_copy")).toContain('href="/admin/backups"');
|
||||
expect(row(html, "restore_drill")).toContain('href="/admin/backups"');
|
||||
expect(row(html, "recovery_kit")).toContain('href="/admin/encryption"');
|
||||
expect(row(html, "offhost_copies")).not.toContain("href=");
|
||||
expect(row(html, "database_dump")).not.toContain("href=");
|
||||
});
|
||||
|
||||
it("does not link to the page already open", () => {
|
||||
pathname.current = "/admin/backups";
|
||||
const html = render(READINESS);
|
||||
expect(row(html, "weekly_copy")).not.toContain("href=");
|
||||
expect(row(html, "recovery_kit")).toContain('href="/admin/encryption"');
|
||||
});
|
||||
|
||||
it("shows a grace period's due date", () => {
|
||||
pathname.current = "/admin";
|
||||
expect(row(render(READINESS), "recovery_kit")).toContain("Due by ");
|
||||
});
|
||||
|
||||
it("keeps a months-old drill failure on the card with its reason", () => {
|
||||
pathname.current = "/admin";
|
||||
const drill = row(render(READINESS), "restore_drill");
|
||||
expect(drill).toContain(
|
||||
"The last restore drill failed: 1 of 4 copies could not be read back.",
|
||||
);
|
||||
expect(drill).toContain("Restore drill: bad tag");
|
||||
expect(drill).toContain("Last drill: ");
|
||||
});
|
||||
|
||||
it("carries the worst state on the badge, and speaks German", () => {
|
||||
pathname.current = "/admin";
|
||||
const html = render(READINESS);
|
||||
expect(html).toContain('data-testid="recovery-readiness-state"');
|
||||
expect(html).toContain('data-state="failing"');
|
||||
expect(render({ ...READINESS, state: "ok" })).toContain(">Ready<");
|
||||
expect(render(READINESS, "de")).toContain("Wiederherstellbarkeit");
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,208 @@
|
||||
"use client";
|
||||
|
||||
/**
|
||||
* `<RecoveryReadinessCard>` — every backup signal on one card: the database
|
||||
* dump, the weekly account copies, the off-host copies, the last restore
|
||||
* drill, the off-host setup and the recovery kit. Each row says its verdict,
|
||||
* the time it is about and where to fix it.
|
||||
*
|
||||
* No time window anywhere: a failed drill stays on this card until a later
|
||||
* drill passes, where the failing-jobs card forgot it after 72 hours. The
|
||||
* verdicts are the server's (`src/lib/recovery/readiness-model.ts`), the same
|
||||
* ones the hourly evaluation pushes to every admin, so the card and the push
|
||||
* cannot disagree.
|
||||
*
|
||||
* Shown on the admin overview and on the Backups page. A row whose fix lives
|
||||
* on the page already open carries no link to itself.
|
||||
*/
|
||||
|
||||
import Link from "next/link";
|
||||
import { usePathname } from "next/navigation";
|
||||
import {
|
||||
AlertTriangle,
|
||||
CheckCircle2,
|
||||
Clock,
|
||||
LifeBuoy,
|
||||
Loader2,
|
||||
MinusCircle,
|
||||
XCircle,
|
||||
} from "lucide-react";
|
||||
|
||||
import { SettingsCardHeader } from "@/components/settings/_card-header";
|
||||
import { SettingsCard } from "@/components/settings/settings-card";
|
||||
import { Badge } from "@/components/ui/badge";
|
||||
import { InlineAction } from "@/components/ui/inline-action";
|
||||
import { QueryErrorRow } from "@/components/ui/query-error-row";
|
||||
import { useFormatters, useTranslations } from "@/lib/i18n/context";
|
||||
import { cn } from "@/lib/utils";
|
||||
|
||||
import {
|
||||
useRecoveryReadiness,
|
||||
type ReadinessSignal,
|
||||
type ReadinessState,
|
||||
type RecoveryReadiness,
|
||||
} from "./use-recovery-readiness";
|
||||
|
||||
const STATE_ICON: Record<
|
||||
ReadinessState,
|
||||
React.ComponentType<{ className?: string; "aria-hidden"?: boolean }>
|
||||
> = {
|
||||
ok: CheckCircle2,
|
||||
attention: AlertTriangle,
|
||||
failing: XCircle,
|
||||
inactive: MinusCircle,
|
||||
};
|
||||
|
||||
const STATE_TONE: Record<ReadinessState, string> = {
|
||||
ok: "text-success",
|
||||
attention: "text-warning",
|
||||
failing: "text-destructive",
|
||||
inactive: "text-muted-foreground",
|
||||
};
|
||||
|
||||
/** A grace period is ok for the verdict, but still a step for the admin. */
|
||||
const isGrace = (signal: ReadinessSignal) => signal.reason.endsWith("_grace");
|
||||
|
||||
function OverallBadge({
|
||||
state,
|
||||
}: {
|
||||
state: Exclude<ReadinessState, "inactive">;
|
||||
}) {
|
||||
const { t } = useTranslations();
|
||||
const label = {
|
||||
ok: t("admin.recoveryReadiness.state.ok"),
|
||||
attention: t("admin.recoveryReadiness.state.attention"),
|
||||
failing: t("admin.recoveryReadiness.state.failing"),
|
||||
}[state];
|
||||
return (
|
||||
<Badge
|
||||
variant="outline"
|
||||
data-testid="recovery-readiness-state"
|
||||
data-state={state}
|
||||
className={cn(
|
||||
state === "attention" && "border-warning/40 text-warning",
|
||||
state === "failing" && "border-destructive/40 text-destructive",
|
||||
)}
|
||||
>
|
||||
{label}
|
||||
</Badge>
|
||||
);
|
||||
}
|
||||
|
||||
function SignalRow({
|
||||
signal,
|
||||
currentPath,
|
||||
}: {
|
||||
signal: ReadinessSignal;
|
||||
currentPath: string | null;
|
||||
}) {
|
||||
const { t } = useTranslations();
|
||||
const fmt = useFormatters();
|
||||
const grace = isGrace(signal);
|
||||
const Icon = grace ? Clock : STATE_ICON[signal.state];
|
||||
const tone = grace ? "text-info" : STATE_TONE[signal.state];
|
||||
const date = signal.at ? fmt.dateTime(new Date(signal.at)) : null;
|
||||
const showFix =
|
||||
signal.state !== "inactive" &&
|
||||
(signal.state !== "ok" || grace) &&
|
||||
signal.fixPath !== currentPath;
|
||||
|
||||
return (
|
||||
<li
|
||||
className="flex items-start gap-3 py-3 first:pt-0 last:pb-0"
|
||||
data-testid={`recovery-signal-${signal.id}`}
|
||||
data-state={signal.state}
|
||||
>
|
||||
<Icon className={cn("mt-0.5 size-4 shrink-0", tone)} aria-hidden />
|
||||
<div className="min-w-0 flex-1 space-y-1">
|
||||
<p className="text-sm font-medium">
|
||||
{t(`admin.recoveryReadiness.signal.${signal.id}`)}
|
||||
</p>
|
||||
<p className="text-sm">
|
||||
{t(
|
||||
`admin.recoveryReadiness.reason.${signal.id}.${signal.reason}`,
|
||||
signal.params,
|
||||
)}
|
||||
</p>
|
||||
{date ? (
|
||||
<p className="text-muted-foreground text-xs">
|
||||
{grace
|
||||
? t("admin.recoveryReadiness.dueBy", { date })
|
||||
: t(`admin.recoveryReadiness.at.${signal.id}`, { date })}
|
||||
</p>
|
||||
) : null}
|
||||
{signal.detail && signal.state !== "ok" ? (
|
||||
<p className="text-muted-foreground text-xs break-words">
|
||||
{signal.detail}
|
||||
</p>
|
||||
) : null}
|
||||
</div>
|
||||
{showFix ? (
|
||||
<InlineAction asChild size="sm" className="shrink-0 font-medium">
|
||||
<Link href={signal.fixPath}>
|
||||
{t(`admin.recoveryReadiness.fix.${signal.id}`)}
|
||||
</Link>
|
||||
</InlineAction>
|
||||
) : null}
|
||||
</li>
|
||||
);
|
||||
}
|
||||
|
||||
/** The list itself, for a readiness that has loaded. */
|
||||
export function RecoveryReadinessList({
|
||||
readiness,
|
||||
currentPath,
|
||||
}: {
|
||||
readiness: RecoveryReadiness;
|
||||
currentPath: string | null;
|
||||
}) {
|
||||
return (
|
||||
<ul className="divide-border divide-y" data-testid="recovery-readiness">
|
||||
{readiness.signals.map((signal) => (
|
||||
<SignalRow key={signal.id} signal={signal} currentPath={currentPath} />
|
||||
))}
|
||||
</ul>
|
||||
);
|
||||
}
|
||||
|
||||
export function RecoveryReadinessCard() {
|
||||
const { t } = useTranslations();
|
||||
const currentPath = usePathname();
|
||||
const { data, isError, error, refetch } = useRecoveryReadiness();
|
||||
|
||||
return (
|
||||
<SettingsCard
|
||||
as="section"
|
||||
aria-labelledby="recovery-readiness-heading"
|
||||
data-testid="recovery-readiness-card"
|
||||
>
|
||||
<SettingsCardHeader
|
||||
icon={LifeBuoy}
|
||||
titleId="recovery-readiness-heading"
|
||||
title={t("admin.recoveryReadiness.title")}
|
||||
description={t("admin.recoveryReadiness.description")}
|
||||
status={data ? <OverallBadge state={data.state} /> : null}
|
||||
/>
|
||||
{data ? (
|
||||
<RecoveryReadinessList readiness={data} currentPath={currentPath} />
|
||||
) : isError ? (
|
||||
<QueryErrorRow
|
||||
inset
|
||||
error={error}
|
||||
message={t("admin.recoveryReadiness.loadError")}
|
||||
onRetry={() => void refetch()}
|
||||
/>
|
||||
) : (
|
||||
<div className="flex items-center gap-2">
|
||||
<Loader2
|
||||
className="text-muted-foreground size-4 animate-spin motion-reduce:animate-none"
|
||||
aria-hidden="true"
|
||||
/>
|
||||
<span className="text-muted-foreground text-sm">
|
||||
{t("admin.recoveryReadiness.loading")}
|
||||
</span>
|
||||
</div>
|
||||
)}
|
||||
</SettingsCard>
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,24 @@
|
||||
"use client";
|
||||
|
||||
import { useQuery } from "@tanstack/react-query";
|
||||
import { apiGet } from "@/lib/api/api-fetch";
|
||||
import { queryKeys } from "@/lib/query-keys";
|
||||
import type { RecoveryReadiness } from "@/lib/recovery/readiness-model";
|
||||
|
||||
export type {
|
||||
ReadinessSignal,
|
||||
ReadinessState,
|
||||
RecoveryReadiness,
|
||||
} from "@/lib/recovery/readiness-model";
|
||||
|
||||
/** `GET /api/admin/recovery-readiness`; admin-only and cookie-only. */
|
||||
export function useRecoveryReadiness() {
|
||||
return useQuery({
|
||||
queryKey: queryKeys.adminRecoveryReadiness(),
|
||||
queryFn: () =>
|
||||
apiGet<RecoveryReadiness>("/api/admin/recovery-readiness", {
|
||||
credentials: "include",
|
||||
}),
|
||||
staleTime: 60_000,
|
||||
});
|
||||
}
|
||||
@@ -50,6 +50,19 @@ describe("summariseBackupSchedule", () => {
|
||||
expect(status.lastRunFailed).toBe(false);
|
||||
});
|
||||
|
||||
it("draws the line where the readiness model does: past nine days", () => {
|
||||
const at = (ms: number) =>
|
||||
summariseBackupSchedule({
|
||||
scheduledCreatedAt: [new Date(NOW.getTime() - ms)],
|
||||
lastRun: null,
|
||||
now: NOW,
|
||||
}).stale;
|
||||
const DAY = 24 * 60 * 60 * 1000;
|
||||
expect(BACKUP_STALE_AFTER_DAYS).toBe(9);
|
||||
expect(at(9 * DAY)).toBe(false);
|
||||
expect(at(9 * DAY + 1)).toBe(true);
|
||||
});
|
||||
|
||||
it("surfaces a failed last run even while the copy is still in window", () => {
|
||||
const status = summariseBackupSchedule({
|
||||
scheduledCreatedAt: [daysAgo(3)],
|
||||
|
||||
@@ -14,7 +14,12 @@ vi.mock("../report-worker-error", () => ({
|
||||
reportWorkerError: vi.fn().mockResolvedValue(undefined),
|
||||
}));
|
||||
|
||||
vi.mock("../restore-drill-outcome", () => ({
|
||||
recordRestoreDrillOutcome: vi.fn().mockResolvedValue(undefined),
|
||||
}));
|
||||
|
||||
import { reportWorkerError } from "../report-worker-error";
|
||||
import { recordRestoreDrillOutcome } from "../restore-drill-outcome";
|
||||
|
||||
const ENC_KEY =
|
||||
"0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef";
|
||||
@@ -257,6 +262,8 @@ describe("handleRestoreDrill", () => {
|
||||
vi.stubEnv("BACKUP_S3_ENDPOINT", "");
|
||||
await handleRestoreDrill([]);
|
||||
expect(reportWorkerError).not.toHaveBeenCalled();
|
||||
// Nothing was checked, so there is no verdict to keep.
|
||||
expect(recordRestoreDrillOutcome).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("pages via reportWorkerError when the drill fails", async () => {
|
||||
@@ -270,5 +277,22 @@ describe("handleRestoreDrill", () => {
|
||||
expect(vi.mocked(reportWorkerError).mock.calls[0][0]).toBe(
|
||||
RESTORE_DRILL_QUEUE,
|
||||
);
|
||||
// The failure is kept for the readiness tile, past pg-boss's retention.
|
||||
expect(recordRestoreDrillOutcome).toHaveBeenCalledTimes(1);
|
||||
expect(vi.mocked(recordRestoreDrillOutcome).mock.calls[0][0]).toMatchObject(
|
||||
{ ok: false, accountsChecked: 0, accountsFailed: 0 },
|
||||
);
|
||||
});
|
||||
|
||||
it("keeps a failed drill's verdict even when keeping it fails", async () => {
|
||||
stubConfigEnv();
|
||||
vi.stubEnv("BACKUP_ENCRYPTION_KEY", "not-a-key");
|
||||
vi.mocked(recordRestoreDrillOutcome).mockRejectedValueOnce(
|
||||
new Error("database away"),
|
||||
);
|
||||
const outcome = await handleRestoreDrill([]);
|
||||
// The drill's own failure is still what the run reports.
|
||||
expect(outcome.ok).toBe(false);
|
||||
expect(reportWorkerError).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
});
|
||||
@@ -23,13 +23,15 @@
|
||||
* alive.
|
||||
*/
|
||||
import type { LastQueueRun } from "@/lib/jobs/job-failures";
|
||||
import { WEEKLY_COPY_ATTENTION_AFTER_DAYS } from "@/lib/recovery/readiness-model";
|
||||
|
||||
/**
|
||||
* How old the newest scheduled copy may get before the page says so. The cron
|
||||
* is weekly, so seven days is normal and eight is a rounding error; ten means
|
||||
* at least one Sunday produced nothing.
|
||||
* is weekly, so seven days is normal and eight is a long week; past nine at
|
||||
* least one Sunday produced nothing. The recovery readiness model draws its
|
||||
* attention line at the same instant, so this page and the push agree.
|
||||
*/
|
||||
export const BACKUP_STALE_AFTER_DAYS = 10;
|
||||
export const BACKUP_STALE_AFTER_DAYS = WEEKLY_COPY_ATTENTION_AFTER_DAYS;
|
||||
|
||||
const DAY_MS = 24 * 60 * 60 * 1000;
|
||||
|
||||
@@ -71,7 +73,9 @@ export function summariseBackupSchedule(args: {
|
||||
// new — the page's empty state already says there is nothing there, and
|
||||
// calling a fresh install "overdue" would train the operator to ignore the
|
||||
// warning that matters.
|
||||
stale: ageDays !== null && ageDays > BACKUP_STALE_AFTER_DAYS,
|
||||
stale:
|
||||
newest !== null &&
|
||||
args.now.getTime() - newest.getTime() > BACKUP_STALE_AFTER_DAYS * DAY_MS,
|
||||
lastRun: args.lastRun,
|
||||
lastRunFailed: args.lastRun !== null && args.lastRun.state !== "completed",
|
||||
};
|
||||
|
||||
@@ -234,6 +234,13 @@ export const JOB_FACT_ALLOWLIST: ReadonlySet<string> = new Set([
|
||||
"restore_drill_plaintext_bytes",
|
||||
"restore_drill_unchecked",
|
||||
"restore_drill_stale",
|
||||
// The hourly recovery readiness evaluation: the overall verdict, how many
|
||||
// signals need attention or are failing, notices sent, admins reached.
|
||||
"recovery_readiness_admins",
|
||||
"recovery_readiness_attention",
|
||||
"recovery_readiness_failing",
|
||||
"recovery_readiness_notices",
|
||||
"recovery_readiness_state",
|
||||
"restore_fail_cause",
|
||||
"restore_failed",
|
||||
"restore_measurements",
|
||||
|
||||
@@ -368,6 +368,9 @@ export const QUEUE_RUNTIME: Readonly<Record<string, QueueRuntime>> = {
|
||||
"geo-backfill": short("At most 500 audit rows per pass."),
|
||||
"geolite2-fetch": short("One database download."),
|
||||
"tls-pin-monitor": short("One TLS handshake."),
|
||||
"recovery-readiness": short(
|
||||
"A few indexed reads, one row per account from two backup ledgers, and at most one notice per signal.",
|
||||
),
|
||||
"integration-watchdog-hourly": short(
|
||||
"A few indexed reads per account with a connection and no provider call; a 14-day history count only for a silent source.",
|
||||
),
|
||||
|
||||
@@ -0,0 +1,12 @@
|
||||
/**
|
||||
* The queue facts of `recovery-readiness.ts`: its name and schedule. They live
|
||||
* apart from the job so the worker registry can read them without loading it.
|
||||
*/
|
||||
export const RECOVERY_READINESS_QUEUE = "recovery-readiness";
|
||||
|
||||
/**
|
||||
* Hourly at :33, a minute no other schedule uses. Hourly is far inside every
|
||||
* threshold the evaluation judges (36 hours for the dump, two nights for the
|
||||
* off-host copies), and a notice is sent once per change whatever the cadence.
|
||||
*/
|
||||
export const RECOVERY_READINESS_CRON = "33 * * * *";
|
||||
@@ -0,0 +1,42 @@
|
||||
/**
|
||||
* The hourly recovery readiness evaluation: read every backup signal, judge
|
||||
* it, and push each change to the admins (`readiness-notify.ts`). The tile on
|
||||
* the admin pages reads the same verdict live; this job is what makes it a
|
||||
* push rather than a page somebody has to open.
|
||||
*
|
||||
* Cheap and bounded: a handful of indexed reads, one row per account for the
|
||||
* two backup ledgers, and at most one notice per signal per run. The next
|
||||
* tick is the retry.
|
||||
*/
|
||||
import type { Job } from "pg-boss";
|
||||
|
||||
import { jobDone, jobFailed, type JobOutcome } from "@/lib/jobs/job-outcome";
|
||||
import { withBackgroundEvent } from "@/lib/logging/background";
|
||||
import { readRecoveryReadiness } from "@/lib/recovery/readiness";
|
||||
import { notifyReadinessChanges } from "@/lib/recovery/readiness-notify";
|
||||
|
||||
export async function handleRecoveryReadiness(
|
||||
jobs: Job<object>[],
|
||||
): Promise<JobOutcome> {
|
||||
void jobs;
|
||||
return withBackgroundEvent("job.recovery_readiness", async (evt) => {
|
||||
try {
|
||||
const readiness = await readRecoveryReadiness();
|
||||
const outcome = await notifyReadinessChanges(readiness);
|
||||
const counts = (state: string) =>
|
||||
readiness.signals.filter((s) => s.state === state).length;
|
||||
const did = {
|
||||
recovery_readiness_state: readiness.state,
|
||||
recovery_readiness_attention: counts("attention"),
|
||||
recovery_readiness_failing: counts("failing"),
|
||||
recovery_readiness_notices: outcome.sent.length,
|
||||
recovery_readiness_admins: outcome.admins,
|
||||
};
|
||||
for (const [key, value] of Object.entries(did)) evt.addMeta(key, value);
|
||||
return jobDone(did);
|
||||
} catch (err) {
|
||||
evt.addWarning(`recovery-readiness failed: ${err}`);
|
||||
return jobFailed("recovery readiness evaluation failed", err);
|
||||
}
|
||||
});
|
||||
}
|
||||
@@ -169,6 +169,10 @@ import {
|
||||
} from "@/lib/jobs/pr-detection";
|
||||
import { PULSE_DAY_MEAN_QUEUE } from "@/lib/jobs/pulse-day-mean";
|
||||
import { cronIsTheRetry } from "@/lib/jobs/reminder/registrar-shared";
|
||||
import {
|
||||
RECOVERY_READINESS_CRON,
|
||||
RECOVERY_READINESS_QUEUE,
|
||||
} from "@/lib/jobs/recovery-readiness-queue";
|
||||
import {
|
||||
RESTORE_DRILL_CRON,
|
||||
RESTORE_DRILL_QUEUE,
|
||||
@@ -343,6 +347,17 @@ const jobs: readonly JobDefinition[] = [
|
||||
load: async () =>
|
||||
(await import("@/lib/jobs/restore-drill")).handleRestoreDrill,
|
||||
}),
|
||||
// Hourly: judge every backup signal and push each change to the admins.
|
||||
// Single-flight and no retries; the next tick is the retry, and the notice
|
||||
// claim keeps two evaluations from sending the same notice.
|
||||
defineJob({
|
||||
queue: RECOVERY_READINESS_QUEUE,
|
||||
cron: RECOVERY_READINESS_CRON,
|
||||
scheduleOptions: cronIsTheRetry,
|
||||
work: { localConcurrency: 1 },
|
||||
load: async () =>
|
||||
(await import("@/lib/jobs/recovery-readiness")).handleRecoveryReadiness,
|
||||
}),
|
||||
// The nightly backstop for the purge an account deletion kicks at once.
|
||||
prune(
|
||||
OFFHOST_PURGE_QUEUE,
|
||||
|
||||
@@ -0,0 +1,54 @@
|
||||
/**
|
||||
* The monthly restore drill's verdict, kept until the next drill replaces it.
|
||||
*
|
||||
* The drill used to say what it found only through `reportWorkerError`
|
||||
* (stderr, and GlitchTip where configured) and through the pg-boss row of its
|
||||
* run. The row is gone after seven days and the admin failing-jobs card reads
|
||||
* 72 hours of it, so a drill that failed on the first of the month was on
|
||||
* screen until the fourth and invisible for the rest of it. One row, written
|
||||
* at the end of every drill that had something to check, is what the recovery
|
||||
* readiness tile and its push read instead, with no window at all.
|
||||
*
|
||||
* A failure stays the verdict until a later drill passes: `lastFailureAt`
|
||||
* newer than `lastSuccessAt` is the rule (`evaluateRestoreDrill`).
|
||||
*/
|
||||
import { prisma } from "@/lib/db";
|
||||
import { redactSecrets } from "@/lib/logging/redact";
|
||||
|
||||
/** The reason is a line on a status tile, not a stack trace. */
|
||||
const MAX_REASON_CHARS = 300;
|
||||
|
||||
export interface RestoreDrillResult {
|
||||
ok: boolean;
|
||||
accountsChecked: number;
|
||||
accountsFailed: number;
|
||||
/** Why it failed; ignored when `ok`. */
|
||||
reason?: string | null;
|
||||
}
|
||||
|
||||
export async function recordRestoreDrillOutcome(
|
||||
result: RestoreDrillResult,
|
||||
at: Date = new Date(),
|
||||
): Promise<void> {
|
||||
const reason = result.ok
|
||||
? null
|
||||
: (() => {
|
||||
const text = redactSecrets((result.reason ?? "").trim());
|
||||
return text.length > MAX_REASON_CHARS
|
||||
? `${text.slice(0, MAX_REASON_CHARS)}…`
|
||||
: text || null;
|
||||
})();
|
||||
const verdict = result.ok
|
||||
? { lastSuccessAt: at }
|
||||
: { lastFailureAt: at, lastFailureReason: reason };
|
||||
const counts = {
|
||||
lastRunAt: at,
|
||||
accountsChecked: result.accountsChecked,
|
||||
accountsFailed: result.accountsFailed,
|
||||
};
|
||||
await prisma.restoreDrillOutcome.upsert({
|
||||
where: { id: "singleton" },
|
||||
create: { id: "singleton", ...counts, ...verdict },
|
||||
update: { ...counts, ...verdict },
|
||||
});
|
||||
}
|
||||
@@ -15,8 +15,10 @@
|
||||
* application key it needs. Each account is reported on its own. It deliberately performs NO database restore — the
|
||||
* drill validates the artefact, not the import path. The result is
|
||||
* surfaced through the wide-event meta on success and through
|
||||
* `reportWorkerError` (stderr + GlitchTip) on any failure, so a broken
|
||||
* backup chain pages the operator eleven months before it matters.
|
||||
* `reportWorkerError` (stderr + GlitchTip) on any failure, and the verdict is
|
||||
* kept (`restore-drill-outcome.ts`) for the recovery readiness tile and its
|
||||
* push to the admins, so a broken backup chain reaches the operator eleven
|
||||
* months before it matters.
|
||||
*
|
||||
* Schedule: 04:11 on the 1st of each month (`11 4 1 * *`), after the
|
||||
* 02:30 nightly upload and the 03:xx cleanup window, on a minute slot
|
||||
@@ -45,6 +47,10 @@ import {
|
||||
type S3Like,
|
||||
} from "@/lib/jobs/offhost-backup";
|
||||
import { reportWorkerError } from "@/lib/jobs/report-worker-error";
|
||||
import {
|
||||
recordRestoreDrillOutcome,
|
||||
type RestoreDrillResult,
|
||||
} from "@/lib/jobs/restore-drill-outcome";
|
||||
import { jobBudget } from "@/lib/jobs/job-budget";
|
||||
import { jobDone, jobFailed, type JobOutcome } from "@/lib/jobs/job-outcome";
|
||||
import { withBackgroundEvent } from "@/lib/logging/background";
|
||||
@@ -254,6 +260,21 @@ export async function runRestoreDrill(
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Keep the verdict for the readiness tile. A failed write is a warning on the
|
||||
* run, never a failed drill: the drill's own reports already went out.
|
||||
*/
|
||||
async function keepOutcome(
|
||||
evt: { addWarning: (message: string) => void },
|
||||
result: RestoreDrillResult,
|
||||
): Promise<void> {
|
||||
try {
|
||||
await recordRestoreDrillOutcome(result);
|
||||
} catch (err) {
|
||||
evt.addWarning(`restore-drill outcome not recorded: ${err}`);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* pg-boss handler. Mirrors the off-host uploader's posture: a
|
||||
* not-configured deployment skips with a wide-event warning (most
|
||||
@@ -323,6 +344,17 @@ export async function handleRestoreDrill(
|
||||
{ dateKey: report.dateKey, unchecked: report.unchecked },
|
||||
);
|
||||
}
|
||||
await keepOutcome(evt, {
|
||||
ok: report.failed.length === 0,
|
||||
accountsChecked: report.accounts.length,
|
||||
accountsFailed: report.failed.length,
|
||||
reason:
|
||||
report.failed.length === 0
|
||||
? null
|
||||
: `${report.failed.length} of ${report.accounts.length} backup objects failed. ${
|
||||
report.failed[0].error ?? report.failed[0].objectKey
|
||||
}`,
|
||||
});
|
||||
if (report.failed.length > 0) {
|
||||
// Already paged above, per account. The queue does not retry: a
|
||||
// retry reads the same objects and would page the same accounts again.
|
||||
@@ -346,6 +378,12 @@ export async function handleRestoreDrill(
|
||||
}
|
||||
evt.addWarning(`restore-drill failed: ${err}`);
|
||||
await reportWorkerError(RESTORE_DRILL_QUEUE, err);
|
||||
await keepOutcome(evt, {
|
||||
ok: false,
|
||||
accountsChecked: 0,
|
||||
accountsFailed: 0,
|
||||
reason: err instanceof Error ? err.message : String(err),
|
||||
});
|
||||
return jobFailed("restore drill failed", err);
|
||||
}
|
||||
});
|
||||
|
||||
@@ -100,6 +100,7 @@ const subsystemSurface = {
|
||||
"step-up-elevation-cleanup": { audience: "system" },
|
||||
"data-backup-offhost": { audience: "system" },
|
||||
"data-restore-drill": { audience: "system" },
|
||||
"recovery-readiness": { audience: "system" },
|
||||
"offhost-backup-purge": { audience: "system" },
|
||||
"backup-restore": { audience: "system" },
|
||||
"host-metric-sample": { audience: "system" },
|
||||
|
||||
@@ -51,6 +51,7 @@ export const NOTIFICATION_TAP_PATHS: Record<EventType, string | null> = {
|
||||
SECURITY_ALERT: "/settings/security",
|
||||
DAILY_BRIEFING: "/",
|
||||
SHARED_RECORD_INTAKE: "/medications",
|
||||
BACKUP_HEALTH: "/admin/backups",
|
||||
};
|
||||
|
||||
const MAX_PATH_LENGTH = 512;
|
||||
|
||||
@@ -125,6 +125,13 @@ export const EVENT_TYPES = [
|
||||
// dispatcher addresses a userId, and under record substitution that userId
|
||||
// is the owner by design). Default ON, see EVENT_DEFAULT_ENABLED.
|
||||
"SHARED_RECORD_INTAKE",
|
||||
// Recovery readiness, for admins only: a backup signal turned to
|
||||
// attention or failing, is still failing a day later, or is back to normal
|
||||
// (`src/lib/recovery/readiness-notify.ts`). Default ON: the whole point is
|
||||
// that a stopped backup is pushed rather than found on a page weeks later.
|
||||
// An admin routes it per channel in the /notifications matrix, where only
|
||||
// admins see the row (`eventTypesForRole`).
|
||||
"BACKUP_HEALTH",
|
||||
] as const;
|
||||
export type EventType = (typeof EVENT_TYPES)[number];
|
||||
|
||||
@@ -214,8 +221,21 @@ export const EVENT_DEFAULT_ENABLED: Record<EventType, boolean> = {
|
||||
// sits in the /notifications matrix from day one, so an owner who finds it
|
||||
// noisy silences it per channel with one switch.
|
||||
SHARED_RECORD_INTAKE: true,
|
||||
// ON by default; see the note on `EVENT_TYPES`. Only admins are ever
|
||||
// addressed, and at most once per state change per signal.
|
||||
BACKUP_HEALTH: true,
|
||||
};
|
||||
|
||||
/** Event types only an admin is ever sent, and only an admin is shown. */
|
||||
export const ADMIN_ONLY_EVENT_TYPES: readonly EventType[] = ["BACKUP_HEALTH"];
|
||||
|
||||
/** The event types a person with `role` can receive, in matrix order. */
|
||||
export function eventTypesForRole(role: string): EventType[] {
|
||||
return role === "ADMIN"
|
||||
? [...EVENT_TYPES]
|
||||
: EVENT_TYPES.filter((type) => !ADMIN_ONLY_EVENT_TYPES.includes(type));
|
||||
}
|
||||
|
||||
export const CHANNEL_TYPE_LABELS: Record<ChannelType, string> = {
|
||||
TELEGRAM: "Telegram",
|
||||
NTFY: "ntfy",
|
||||
|
||||
@@ -0,0 +1,424 @@
|
||||
/**
|
||||
* The recovery readiness thresholds, one signal at a time, on both sides of
|
||||
* every line the model draws.
|
||||
*
|
||||
* Mutation check: move any threshold by one unit (35 h for the dump, 8 or 13
|
||||
* days for the weekly copy, 6 days of setup grace) and the matching
|
||||
* "just inside" or "just past" case goes red.
|
||||
*/
|
||||
import { describe, expect, it } from "vitest";
|
||||
|
||||
import type { DatabaseBackupHealth } from "@/lib/recovery/database-dump-health";
|
||||
import {
|
||||
DUMP_FAILING_AFTER_HOURS,
|
||||
evaluateDatabaseDump,
|
||||
evaluateOffhostCopies,
|
||||
evaluateOffhostSetup,
|
||||
evaluateRecoveryKit,
|
||||
evaluateRecoveryReadiness,
|
||||
evaluateRestoreDrill,
|
||||
evaluateWeeklyCopy,
|
||||
READINESS_FIX_PATHS,
|
||||
READINESS_SIGNALS,
|
||||
SETUP_GRACE_DAYS,
|
||||
WEEKLY_COPY_ATTENTION_AFTER_DAYS,
|
||||
WEEKLY_COPY_FAILING_AFTER_DAYS,
|
||||
type ReadinessInputs,
|
||||
} from "@/lib/recovery/readiness-model";
|
||||
|
||||
const NOW = new Date("2026-10-25T12:00:00Z");
|
||||
const HOUR = 3_600_000;
|
||||
const DAY = 24 * HOUR;
|
||||
const ago = (ms: number) => new Date(NOW.getTime() - ms);
|
||||
|
||||
/** A dump report in the dump module's own shape. */
|
||||
function dump(
|
||||
over: Partial<DatabaseBackupHealth> & {
|
||||
lastGoodAt?: Date | null;
|
||||
lastFailureAt?: Date | null;
|
||||
lastFailureReason?: string;
|
||||
} = {},
|
||||
): DatabaseBackupHealth {
|
||||
const {
|
||||
lastGoodAt = ago(2 * HOUR),
|
||||
lastFailureAt = null,
|
||||
lastFailureReason,
|
||||
...rest
|
||||
} = over;
|
||||
return {
|
||||
enabled: true,
|
||||
directory: "/data/db-backups",
|
||||
localDirWritable: true,
|
||||
offhostConfigured: false,
|
||||
lastGood: lastGoodAt
|
||||
? {
|
||||
at: lastGoodAt.toISOString(),
|
||||
fileName: "healthlog.dump.enc",
|
||||
sizeBytes: 1024,
|
||||
local: true,
|
||||
offhost: false,
|
||||
}
|
||||
: null,
|
||||
lastFailure: lastFailureAt
|
||||
? {
|
||||
at: lastFailureAt.toISOString(),
|
||||
error: lastFailureReason ?? "failed",
|
||||
}
|
||||
: null,
|
||||
...rest,
|
||||
};
|
||||
}
|
||||
|
||||
function inputs(over: Partial<ReadinessInputs> = {}): ReadinessInputs {
|
||||
return {
|
||||
now: NOW,
|
||||
firstAdminAt: ago(30 * DAY),
|
||||
dump: null,
|
||||
dumpWatchedSince: null,
|
||||
weekly: { accounts: [{ createdAt: ago(60 * DAY), lastCopyAt: ago(DAY) }] },
|
||||
offhost: {
|
||||
configured: true,
|
||||
accounts: [
|
||||
{ lastAttemptAt: ago(10 * HOUR), lastSuccessAt: ago(10 * HOUR) },
|
||||
],
|
||||
},
|
||||
drill: null,
|
||||
kit: { confirmedAt: ago(10 * DAY), unconfirmed: [] },
|
||||
...over,
|
||||
};
|
||||
}
|
||||
|
||||
describe("database dump", () => {
|
||||
const evalDump = (d: ReadinessInputs["dump"], watched: Date | null = null) =>
|
||||
evaluateDatabaseDump({ now: NOW, dump: d, dumpWatchedSince: watched });
|
||||
|
||||
it("does not apply while no dump is scheduled, or while it is switched off", () => {
|
||||
expect(evalDump(null)).toMatchObject({
|
||||
state: "inactive",
|
||||
reason: "not_available",
|
||||
});
|
||||
expect(evalDump(dump({ enabled: false }))).toMatchObject({
|
||||
state: "inactive",
|
||||
reason: "disabled",
|
||||
});
|
||||
});
|
||||
|
||||
it("fails when its status cannot be read", () => {
|
||||
expect(evalDump("unreadable")).toMatchObject({
|
||||
state: "failing",
|
||||
reason: "unreadable",
|
||||
});
|
||||
});
|
||||
|
||||
it("is ok up to 36 hours and failing past them", () => {
|
||||
const inside = evalDump(
|
||||
dump({ lastGoodAt: ago(DUMP_FAILING_AFTER_HOURS * HOUR - 1) }),
|
||||
);
|
||||
expect(inside).toMatchObject({ state: "ok", reason: "fresh" });
|
||||
const past = evalDump(
|
||||
dump({ lastGoodAt: ago(DUMP_FAILING_AFTER_HOURS * HOUR + 1) }),
|
||||
);
|
||||
expect(past).toMatchObject({
|
||||
state: "failing",
|
||||
reason: "stale",
|
||||
params: { hours: 36 },
|
||||
});
|
||||
expect(past.at).toBe(
|
||||
ago(DUMP_FAILING_AFTER_HOURS * HOUR + 1).toISOString(),
|
||||
);
|
||||
});
|
||||
|
||||
it("calls a recent failure after a recent dump attention, with the reason", () => {
|
||||
const s = evalDump(
|
||||
dump({ lastFailureAt: ago(HOUR), lastFailureReason: "pg_dump exited 1" }),
|
||||
);
|
||||
expect(s).toMatchObject({
|
||||
state: "attention",
|
||||
reason: "last_failed",
|
||||
detail: "pg_dump exited 1",
|
||||
});
|
||||
});
|
||||
|
||||
it("fails when a dump can be kept nowhere, and accepts a bucket-only setup", () => {
|
||||
expect(evalDump(dump({ localDirWritable: false }))).toMatchObject({
|
||||
state: "failing",
|
||||
reason: "nowhere_to_write",
|
||||
params: { directory: "/data/db-backups" },
|
||||
});
|
||||
expect(
|
||||
evalDump(dump({ localDirWritable: false, offhostConfigured: true }))
|
||||
.state,
|
||||
).toBe("ok");
|
||||
});
|
||||
|
||||
it("gives a newly switched-on dump 36 hours before calling it failing", () => {
|
||||
const fresh = dump({ lastGoodAt: null });
|
||||
expect(evalDump(fresh, ago(HOUR))).toMatchObject({
|
||||
state: "ok",
|
||||
reason: "first_pending",
|
||||
});
|
||||
expect(evalDump(fresh, null)).toMatchObject({
|
||||
state: "ok",
|
||||
reason: "first_pending",
|
||||
});
|
||||
expect(evalDump(fresh, ago(37 * HOUR))).toMatchObject({
|
||||
state: "failing",
|
||||
reason: "never_ran",
|
||||
});
|
||||
expect(
|
||||
evalDump(dump({ lastGoodAt: null, lastFailureAt: ago(HOUR) }), ago(HOUR)),
|
||||
).toMatchObject({ state: "failing", reason: "never_succeeded" });
|
||||
});
|
||||
});
|
||||
|
||||
describe("weekly copy", () => {
|
||||
const weekly = (
|
||||
...accounts: Array<{ createdAt: Date; lastCopyAt: Date | null }>
|
||||
) => evaluateWeeklyCopy({ now: NOW, weekly: { accounts } });
|
||||
|
||||
it("is attention past 9 days and failing past 14, judged by the worst account", () => {
|
||||
const fresh = { createdAt: ago(90 * DAY), lastCopyAt: ago(DAY) };
|
||||
expect(
|
||||
weekly(fresh, { createdAt: ago(90 * DAY), lastCopyAt: ago(9 * DAY - 1) })
|
||||
.state,
|
||||
).toBe("ok");
|
||||
expect(
|
||||
weekly(fresh, {
|
||||
createdAt: ago(90 * DAY),
|
||||
lastCopyAt: ago(WEEKLY_COPY_ATTENTION_AFTER_DAYS * DAY + 1),
|
||||
}),
|
||||
).toMatchObject({
|
||||
state: "attention",
|
||||
reason: "stale",
|
||||
params: { days: 9, accounts: 1 },
|
||||
});
|
||||
expect(
|
||||
weekly(fresh, {
|
||||
createdAt: ago(90 * DAY),
|
||||
lastCopyAt: ago(WEEKLY_COPY_FAILING_AFTER_DAYS * DAY - 1),
|
||||
}).state,
|
||||
).toBe("attention");
|
||||
expect(
|
||||
weekly(fresh, {
|
||||
createdAt: ago(90 * DAY),
|
||||
lastCopyAt: ago(WEEKLY_COPY_FAILING_AFTER_DAYS * DAY + 1),
|
||||
}),
|
||||
).toMatchObject({ state: "failing", params: { days: 14 } });
|
||||
});
|
||||
|
||||
it("ages an account without any copy from its creation", () => {
|
||||
expect(weekly({ createdAt: ago(3 * DAY), lastCopyAt: null })).toMatchObject(
|
||||
{
|
||||
state: "ok",
|
||||
reason: "first_pending",
|
||||
},
|
||||
);
|
||||
expect(
|
||||
weekly({ createdAt: ago(20 * DAY), lastCopyAt: null }),
|
||||
).toMatchObject({
|
||||
state: "failing",
|
||||
reason: "never",
|
||||
});
|
||||
});
|
||||
|
||||
it("does not apply without accounts", () => {
|
||||
expect(weekly().state).toBe("inactive");
|
||||
});
|
||||
});
|
||||
|
||||
describe("off-host copies", () => {
|
||||
const offhost = (
|
||||
configured: boolean,
|
||||
...accounts: Array<{
|
||||
lastAttemptAt: Date | null;
|
||||
lastSuccessAt: Date | null;
|
||||
}>
|
||||
) => evaluateOffhostCopies({ now: NOW, offhost: { configured, accounts } });
|
||||
|
||||
it("does not apply while no bucket is configured", () => {
|
||||
expect(offhost(false).state).toBe("inactive");
|
||||
});
|
||||
|
||||
it("calls any account more than two nights behind attention", () => {
|
||||
const fresh = {
|
||||
lastAttemptAt: ago(5 * HOUR),
|
||||
lastSuccessAt: ago(5 * HOUR),
|
||||
};
|
||||
// One missed night (inside two periods plus the grace) is not yet news.
|
||||
expect(
|
||||
offhost(true, fresh, {
|
||||
lastAttemptAt: ago(HOUR),
|
||||
lastSuccessAt: ago(40 * HOUR),
|
||||
}).state,
|
||||
).toBe("ok");
|
||||
expect(
|
||||
offhost(true, fresh, {
|
||||
lastAttemptAt: ago(HOUR),
|
||||
lastSuccessAt: ago(2 * DAY + 7 * HOUR),
|
||||
}),
|
||||
).toMatchObject({
|
||||
state: "attention",
|
||||
reason: "stale",
|
||||
params: { accounts: 1 },
|
||||
});
|
||||
});
|
||||
|
||||
it("counts an account a run reached and never wrote as behind", () => {
|
||||
expect(
|
||||
offhost(true, { lastAttemptAt: ago(HOUR), lastSuccessAt: null }),
|
||||
).toMatchObject({ state: "attention", params: { accounts: 1 } });
|
||||
});
|
||||
|
||||
it("waits for the first run rather than calling the cohort behind", () => {
|
||||
expect(
|
||||
offhost(true, { lastAttemptAt: null, lastSuccessAt: null }),
|
||||
).toMatchObject({
|
||||
state: "ok",
|
||||
reason: "first_pending",
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe("restore drill", () => {
|
||||
const drill = (d: ReadinessInputs["drill"], configured = true) =>
|
||||
evaluateRestoreDrill({ offhost: { configured, accounts: [] }, drill: d });
|
||||
const run = {
|
||||
lastRunAt: ago(5 * DAY),
|
||||
lastSuccessAt: null as Date | null,
|
||||
lastFailureAt: null as Date | null,
|
||||
lastFailureReason: null as string | null,
|
||||
accountsChecked: 3,
|
||||
accountsFailed: 0,
|
||||
};
|
||||
|
||||
it("stays failing after a failure, with no time window, until a later success", () => {
|
||||
const failed = {
|
||||
...run,
|
||||
lastFailureAt: ago(60 * DAY),
|
||||
accountsFailed: 1,
|
||||
lastFailureReason: "bad tag",
|
||||
};
|
||||
expect(drill(failed)).toMatchObject({
|
||||
state: "failing",
|
||||
reason: "failed",
|
||||
params: { failed: 1, checked: 3 },
|
||||
detail: "bad tag",
|
||||
});
|
||||
expect(drill({ ...failed, lastSuccessAt: ago(61 * DAY) }).state).toBe(
|
||||
"failing",
|
||||
);
|
||||
expect(drill({ ...failed, lastSuccessAt: ago(DAY) })).toMatchObject({
|
||||
state: "ok",
|
||||
reason: "passed",
|
||||
});
|
||||
});
|
||||
|
||||
it("names a run that checked nothing", () => {
|
||||
expect(
|
||||
drill({ ...run, accountsChecked: 0, lastFailureAt: ago(DAY) }).reason,
|
||||
).toBe("failed_run");
|
||||
});
|
||||
|
||||
it("does not apply without a bucket, and is ok before its first run", () => {
|
||||
expect(drill(null, false).state).toBe("inactive");
|
||||
expect(drill(null)).toMatchObject({ state: "ok", reason: "not_run_yet" });
|
||||
});
|
||||
});
|
||||
|
||||
describe("off-host setup", () => {
|
||||
const setup = (configured: boolean, firstAdminAt: Date | null) =>
|
||||
evaluateOffhostSetup({
|
||||
now: NOW,
|
||||
firstAdminAt,
|
||||
offhost: { configured, accounts: [] },
|
||||
});
|
||||
|
||||
it("asks for a bucket 7 days after the first admin exists", () => {
|
||||
expect(setup(false, ago(SETUP_GRACE_DAYS * DAY - 1))).toMatchObject({
|
||||
state: "ok",
|
||||
reason: "not_configured_grace",
|
||||
});
|
||||
expect(setup(false, ago(SETUP_GRACE_DAYS * DAY + 1))).toMatchObject({
|
||||
state: "attention",
|
||||
reason: "not_configured",
|
||||
});
|
||||
expect(setup(true, ago(90 * DAY)).state).toBe("ok");
|
||||
expect(setup(false, null).state).toBe("inactive");
|
||||
});
|
||||
});
|
||||
|
||||
describe("recovery kit", () => {
|
||||
const kit = (
|
||||
confirmedAt: Date | null,
|
||||
unconfirmed: Array<{
|
||||
kind: "application" | "backup" | "token";
|
||||
id: string;
|
||||
}>,
|
||||
firstAdminAt: Date | null = ago(30 * DAY),
|
||||
) =>
|
||||
evaluateRecoveryKit({
|
||||
now: NOW,
|
||||
firstAdminAt,
|
||||
kit: { confirmedAt, unconfirmed },
|
||||
});
|
||||
|
||||
it("is ok while the confirmation covers every key", () => {
|
||||
expect(kit(ago(DAY), [])).toMatchObject({
|
||||
state: "ok",
|
||||
reason: "confirmed",
|
||||
});
|
||||
});
|
||||
|
||||
it("names a needed key nobody confirmed, at once once a kit was confirmed", () => {
|
||||
expect(kit(ago(DAY), [{ kind: "application", id: "v1" }])).toMatchObject({
|
||||
state: "attention",
|
||||
reason: "out_of_date",
|
||||
params: { keys: "v1", count: 1 },
|
||||
});
|
||||
});
|
||||
|
||||
it("gives a never-confirmed kit 7 days after setup", () => {
|
||||
const keys = [
|
||||
{ kind: "application" as const, id: "v1" },
|
||||
{ kind: "token" as const, id: "API_TOKEN_HMAC_KEY" },
|
||||
];
|
||||
expect(kit(null, keys, ago(SETUP_GRACE_DAYS * DAY - 1))).toMatchObject({
|
||||
state: "ok",
|
||||
reason: "not_confirmed_grace",
|
||||
});
|
||||
expect(kit(null, keys, ago(SETUP_GRACE_DAYS * DAY + 1))).toMatchObject({
|
||||
state: "attention",
|
||||
reason: "not_confirmed",
|
||||
params: { keys: "v1, API_TOKEN_HMAC_KEY", count: 2 },
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe("the whole verdict", () => {
|
||||
it("is the worst applying signal, and inactive ones never count", () => {
|
||||
expect(evaluateRecoveryReadiness(inputs()).state).toBe("ok");
|
||||
expect(
|
||||
evaluateRecoveryReadiness(
|
||||
inputs({ dump: dump({ lastGoodAt: ago(40 * HOUR) }) }),
|
||||
).state,
|
||||
).toBe("failing");
|
||||
expect(
|
||||
evaluateRecoveryReadiness(
|
||||
inputs({
|
||||
kit: {
|
||||
confirmedAt: ago(DAY),
|
||||
unconfirmed: [{ kind: "backup", id: "abc" }],
|
||||
},
|
||||
}),
|
||||
).state,
|
||||
).toBe("attention");
|
||||
});
|
||||
|
||||
it("reports every signal once, in order, each with a fix path", () => {
|
||||
const r = evaluateRecoveryReadiness(inputs());
|
||||
expect(r.signals.map((s) => s.id)).toEqual([...READINESS_SIGNALS]);
|
||||
for (const s of r.signals)
|
||||
expect(s.fixPath).toBe(READINESS_FIX_PATHS[s.id]);
|
||||
expect(r.evaluatedAt).toBe(NOW.toISOString());
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,287 @@
|
||||
/**
|
||||
* The notice state machine: once per state change per signal, a recovery
|
||||
* notice once, a reminder at most once a day while failing, nothing else.
|
||||
*
|
||||
* Two layers. `decideNotice` is pure and pinned case by case.
|
||||
* `notifyReadinessChanges` runs against an in-memory stand-in for its one
|
||||
* table that honours the compare-and-set exactly as Postgres would (a row
|
||||
* moves only from the values the caller read), so the hourly sequence and the
|
||||
* two-workers race are proven on the real claim logic rather than assumed.
|
||||
*
|
||||
* Mutation check: drop the `last !== current` comparison and the "same state
|
||||
* twice" case sends twice; drop the CAS `where` on `notifiedState` and the
|
||||
* race sends two notices.
|
||||
*/
|
||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
|
||||
const store = vi.hoisted(() => ({
|
||||
notices: new Map<
|
||||
string,
|
||||
{
|
||||
signal: string;
|
||||
notifiedState: string | null;
|
||||
notifiedAt: Date | null;
|
||||
watchedSince: Date | null;
|
||||
}
|
||||
>(),
|
||||
admins: [] as Array<{ id: string }>,
|
||||
}));
|
||||
|
||||
const same = (a: Date | null, b: Date | null | undefined) =>
|
||||
(a?.getTime() ?? null) === (b?.getTime() ?? null);
|
||||
|
||||
vi.mock("@/lib/db", () => ({
|
||||
prisma: {
|
||||
user: { findMany: vi.fn(async () => store.admins) },
|
||||
recoveryReadinessNotice: {
|
||||
createMany: vi.fn(
|
||||
async ({ data }: { data: Array<{ signal: string }> }) => {
|
||||
for (const { signal } of data) {
|
||||
if (!store.notices.has(signal)) {
|
||||
store.notices.set(signal, {
|
||||
signal,
|
||||
notifiedState: null,
|
||||
notifiedAt: null,
|
||||
watchedSince: null,
|
||||
});
|
||||
}
|
||||
}
|
||||
return { count: data.length };
|
||||
},
|
||||
),
|
||||
findMany: vi.fn(async () =>
|
||||
[...store.notices.values()].map((row) => ({ ...row })),
|
||||
),
|
||||
updateMany: vi.fn(
|
||||
async ({
|
||||
where,
|
||||
data,
|
||||
}: {
|
||||
where: {
|
||||
signal: string;
|
||||
notifiedState?: string | null;
|
||||
notifiedAt?: Date | null;
|
||||
watchedSince?: Date | null;
|
||||
};
|
||||
data: Record<string, unknown>;
|
||||
}) => {
|
||||
const row = store.notices.get(where.signal);
|
||||
if (!row) return { count: 0 };
|
||||
if (
|
||||
"notifiedState" in where &&
|
||||
row.notifiedState !== where.notifiedState
|
||||
)
|
||||
return { count: 0 };
|
||||
if ("notifiedAt" in where && !same(row.notifiedAt, where.notifiedAt))
|
||||
return { count: 0 };
|
||||
if (
|
||||
"watchedSince" in where &&
|
||||
!same(row.watchedSince, where.watchedSince)
|
||||
)
|
||||
return { count: 0 };
|
||||
Object.assign(row, data);
|
||||
return { count: 1 };
|
||||
},
|
||||
),
|
||||
},
|
||||
},
|
||||
}));
|
||||
|
||||
const dispatch = vi.hoisted(() =>
|
||||
vi.fn(async (opts: Record<string, unknown>) => {
|
||||
void opts;
|
||||
}),
|
||||
);
|
||||
vi.mock("@/lib/notifications/dispatch-localised", () => ({
|
||||
dispatchLocalisedNotification: dispatch,
|
||||
}));
|
||||
|
||||
import {
|
||||
decideNotice,
|
||||
FAILING_REMINDER_HOURS,
|
||||
notifyReadinessChanges,
|
||||
} from "@/lib/recovery/readiness-notify";
|
||||
import {
|
||||
READINESS_FIX_PATHS,
|
||||
READINESS_SIGNALS,
|
||||
type ReadinessSignal,
|
||||
type ReadinessState,
|
||||
type RecoveryReadiness,
|
||||
} from "@/lib/recovery/readiness-model";
|
||||
|
||||
const T0 = new Date("2026-10-25T12:00:00Z");
|
||||
const HOUR = 3_600_000;
|
||||
const at = (hours: number) => new Date(T0.getTime() + hours * HOUR);
|
||||
|
||||
describe("decideNotice", () => {
|
||||
it("announces a change to attention or failing once", () => {
|
||||
expect(decideNotice(null, "attention", T0)).toEqual({
|
||||
send: "problem",
|
||||
next: { notifiedState: "attention", notifiedAt: T0 },
|
||||
});
|
||||
const told = { notifiedState: "attention" as const, notifiedAt: T0 };
|
||||
expect(decideNotice(told, "attention", at(30)).send).toBeNull();
|
||||
expect(decideNotice(told, "failing", at(1)).send).toBe("problem");
|
||||
});
|
||||
|
||||
it("reminds a failing signal once a day, not sooner", () => {
|
||||
const told = { notifiedState: "failing" as const, notifiedAt: T0 };
|
||||
expect(
|
||||
decideNotice(told, "failing", at(FAILING_REMINDER_HOURS - 1)).send,
|
||||
).toBeNull();
|
||||
expect(decideNotice(told, "failing", at(FAILING_REMINDER_HOURS))).toEqual({
|
||||
send: "reminder",
|
||||
next: {
|
||||
notifiedState: "failing",
|
||||
notifiedAt: at(FAILING_REMINDER_HOURS),
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
it("announces a recovery once, and says nothing about an ok that was never broken", () => {
|
||||
const told = { notifiedState: "failing" as const, notifiedAt: T0 };
|
||||
expect(decideNotice(told, "ok", at(2)).send).toBe("recovered");
|
||||
expect(
|
||||
decideNotice({ notifiedState: "ok", notifiedAt: at(2) }, "ok", at(3)),
|
||||
).toEqual({
|
||||
send: null,
|
||||
next: null,
|
||||
});
|
||||
expect(decideNotice(null, "ok", T0)).toEqual({ send: null, next: null });
|
||||
});
|
||||
|
||||
it("forgets silently when a signal stops applying", () => {
|
||||
expect(
|
||||
decideNotice(
|
||||
{ notifiedState: "failing", notifiedAt: T0 },
|
||||
"inactive",
|
||||
at(1),
|
||||
),
|
||||
).toEqual({ send: null, next: { notifiedState: null, notifiedAt: null } });
|
||||
expect(decideNotice(null, "inactive", T0)).toEqual({
|
||||
send: null,
|
||||
next: null,
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
function readiness(
|
||||
states: Partial<Record<ReadinessSignal["id"], [ReadinessState, string]>>,
|
||||
now: Date,
|
||||
): RecoveryReadiness {
|
||||
return {
|
||||
state: "ok",
|
||||
evaluatedAt: now.toISOString(),
|
||||
signals: READINESS_SIGNALS.map((id) => {
|
||||
const [state, reason] = states[id] ?? ["ok", "fresh"];
|
||||
return {
|
||||
id,
|
||||
state,
|
||||
reason,
|
||||
at: null,
|
||||
params: { days: 12, accounts: 1 },
|
||||
detail: null,
|
||||
fixPath: READINESS_FIX_PATHS[id],
|
||||
};
|
||||
}),
|
||||
};
|
||||
}
|
||||
|
||||
const sent = () =>
|
||||
dispatch.mock.calls.map(([opts]) => {
|
||||
const o = opts as unknown as {
|
||||
userId: string;
|
||||
titleKey: string;
|
||||
messageKey: string;
|
||||
};
|
||||
return `${o.userId} ${o.titleKey.split(".").at(-1)} ${o.messageKey}`;
|
||||
});
|
||||
|
||||
describe("notifyReadinessChanges", () => {
|
||||
beforeEach(() => {
|
||||
store.notices.clear();
|
||||
store.admins = [{ id: "admin-a" }, { id: "admin-b" }];
|
||||
dispatch.mockClear();
|
||||
});
|
||||
|
||||
it("sends each notice to every admin, through the backup event", async () => {
|
||||
await notifyReadinessChanges(
|
||||
readiness({ weekly_copy: ["attention", "stale"] }, T0),
|
||||
);
|
||||
expect(sent()).toEqual([
|
||||
"admin-a titleAttention admin.recoveryReadiness.reason.weekly_copy.stale",
|
||||
"admin-b titleAttention admin.recoveryReadiness.reason.weekly_copy.stale",
|
||||
]);
|
||||
const first = dispatch.mock.calls[0][0] as unknown as {
|
||||
eventType: string;
|
||||
params: Record<string, unknown>;
|
||||
metadata: Record<string, unknown>;
|
||||
};
|
||||
expect(first.eventType).toBe("BACKUP_HEALTH");
|
||||
expect(first.params).toEqual({ days: 12, accounts: 1 });
|
||||
expect(first.metadata).toMatchObject({
|
||||
url: "/admin/backups",
|
||||
signal: "weekly_copy",
|
||||
});
|
||||
});
|
||||
|
||||
it("runs an hourly sequence without nagging: change, silence, escalation, daily reminder, recovery", async () => {
|
||||
store.admins = [{ id: "admin-a" }];
|
||||
const run = (h: number, state: ReadinessState, reason: string) =>
|
||||
notifyReadinessChanges(
|
||||
readiness({ restore_drill: [state, reason] }, at(h)),
|
||||
);
|
||||
|
||||
await run(0, "attention", "failed");
|
||||
for (let h = 1; h < 5; h++) await run(h, "attention", "failed");
|
||||
expect(sent()).toHaveLength(1);
|
||||
|
||||
await run(5, "failing", "failed");
|
||||
for (let h = 6; h < 5 + FAILING_REMINDER_HOURS; h++)
|
||||
await run(h, "failing", "failed");
|
||||
expect(sent()).toHaveLength(2);
|
||||
|
||||
await run(5 + FAILING_REMINDER_HOURS, "failing", "failed");
|
||||
await run(6 + FAILING_REMINDER_HOURS, "failing", "failed");
|
||||
expect(sent()).toHaveLength(3);
|
||||
expect(sent()[2]).toContain("titleReminder");
|
||||
|
||||
await run(40, "ok", "passed");
|
||||
await run(41, "ok", "passed");
|
||||
expect(sent()).toHaveLength(4);
|
||||
expect(sent()[3]).toBe(
|
||||
"admin-a titleRecovered admin.recoveryReadiness.recovered.restore_drill",
|
||||
);
|
||||
});
|
||||
|
||||
it("sends one notice when two evaluations race on the same row", async () => {
|
||||
const r = readiness({ recovery_kit: ["attention", "out_of_date"] }, T0);
|
||||
await Promise.all([notifyReadinessChanges(r), notifyReadinessChanges(r)]);
|
||||
expect(sent()).toHaveLength(2); // one notice, two admins
|
||||
});
|
||||
|
||||
it("claims nothing while there is no admin to tell", async () => {
|
||||
store.admins = [];
|
||||
const r = readiness({ offhost_setup: ["attention", "not_configured"] }, T0);
|
||||
await notifyReadinessChanges(r);
|
||||
expect(store.notices.get("offhost_setup")?.notifiedState).toBeNull();
|
||||
store.admins = [{ id: "admin-a" }];
|
||||
await notifyReadinessChanges(r);
|
||||
expect(sent()).toHaveLength(1);
|
||||
});
|
||||
|
||||
it("remembers since when a signal applies, and forgets it when it stops", async () => {
|
||||
await notifyReadinessChanges(
|
||||
readiness({ database_dump: ["ok", "fresh"] }, T0),
|
||||
);
|
||||
expect(store.notices.get("database_dump")?.watchedSince).toEqual(T0);
|
||||
await notifyReadinessChanges(
|
||||
readiness({ database_dump: ["ok", "fresh"] }, at(5)),
|
||||
);
|
||||
expect(store.notices.get("database_dump")?.watchedSince).toEqual(T0);
|
||||
await notifyReadinessChanges(
|
||||
readiness({ database_dump: ["inactive", "disabled"] }, at(6)),
|
||||
);
|
||||
expect(store.notices.get("database_dump")?.watchedSince).toBeNull();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,50 @@
|
||||
/**
|
||||
* The scheduled database dump, as the recovery readiness model sees it.
|
||||
*
|
||||
* The dump is the one artifact that rebuilds a lost host: it holds every
|
||||
* account, every grant and every credential, and a schema change cannot leave
|
||||
* anything out of it. Its health is a signal of its own in the readiness
|
||||
* model, read through a provider so the model does not reach into the dump's
|
||||
* module and a test can hand in any state it wants.
|
||||
*
|
||||
* A provider that answers `null` means "this build does not schedule a dump",
|
||||
* which the model reports as not available rather than as healthy or broken.
|
||||
*/
|
||||
|
||||
/**
|
||||
* What the dump module reports about its own schedule. Mirrors
|
||||
* `DatabaseBackupHealth` in `src/lib/database-backup/health.ts` field for
|
||||
* field, so the provider below becomes that module's
|
||||
* `getDatabaseBackupHealth` without a translation.
|
||||
*/
|
||||
export interface DatabaseBackupHealth {
|
||||
/** `DB_BACKUP_ENABLED` is not off. */
|
||||
enabled: boolean;
|
||||
/** The directory dumps are kept in (`DB_BACKUP_DIR`). */
|
||||
directory: string;
|
||||
/** This process can write there. False means dumps only go off-host. */
|
||||
localDirWritable: boolean;
|
||||
/** The off-host bucket is configured, or a dump has been uploaded to it. */
|
||||
offhostConfigured: boolean;
|
||||
/** The newest dump that was kept on the server or reached the bucket. */
|
||||
lastGood: {
|
||||
at: string;
|
||||
fileName: string;
|
||||
sizeBytes: number | null;
|
||||
local: boolean;
|
||||
offhost: boolean;
|
||||
} | null;
|
||||
/** The newest run that failed, or that a dying process left unfinished. */
|
||||
lastFailure: { at: string; error: string } | null;
|
||||
}
|
||||
|
||||
export type DatabaseBackupHealthProvider = (
|
||||
now?: Date,
|
||||
) => Promise<DatabaseBackupHealth | null>;
|
||||
|
||||
/**
|
||||
* The provider every caller uses unless it passes its own: the dump's health,
|
||||
* or `null` while no dump is scheduled by this build.
|
||||
*/
|
||||
export const readDatabaseBackupHealth: DatabaseBackupHealthProvider =
|
||||
async () => null;
|
||||
@@ -0,0 +1,422 @@
|
||||
/**
|
||||
* Recovery readiness: can this instance be put back after a loss, judged from
|
||||
* every fact the server holds about its own copies, in one place.
|
||||
*
|
||||
* Every one of these facts used to be a pull, on a page nobody opens until the
|
||||
* day it matters. The weekly copy stopped for 46 days and showed a perfectly
|
||||
* ordinary row. A failed monthly drill was on the failing-jobs card three days
|
||||
* in thirty. An off-host bucket that was never configured showed only on the
|
||||
* backups page, and the key step confirmed one key of the several a restore
|
||||
* needs. This module turns them into six signals with one verdict each, which
|
||||
* the readiness tile shows and the hourly evaluation pushes to every admin.
|
||||
*
|
||||
* Pure: the inputs are read elsewhere (`readiness.ts`), so each threshold is a
|
||||
* unit test away and the tile and the push cannot disagree about a verdict.
|
||||
*
|
||||
* States:
|
||||
* ok — nothing to do. A grace period (`*_grace` reasons) is ok too:
|
||||
* the admin is shown the step, nobody is pushed about it yet.
|
||||
* attention — something will hurt on the day of a restore; worth a push.
|
||||
* failing — the instance cannot be restored the way it is meant to be.
|
||||
* inactive — the signal does not apply here (no dump in this build, no
|
||||
* off-host bucket for the drill). Never pushed, never counts
|
||||
* towards the overall verdict.
|
||||
*/
|
||||
import {
|
||||
classifyOffhostBackup,
|
||||
type OffhostBackupFreshness,
|
||||
} from "@/lib/jobs/offhost-backup-freshness";
|
||||
import type { DatabaseBackupHealth } from "@/lib/recovery/database-dump-health";
|
||||
import type { KitEntry } from "@/lib/recovery/kit-rules";
|
||||
|
||||
const HOUR_MS = 60 * 60 * 1000;
|
||||
const DAY_MS = 24 * HOUR_MS;
|
||||
|
||||
/** A dump older than this, while dumps are on, means no usable dump. */
|
||||
export const DUMP_FAILING_AFTER_HOURS = 36;
|
||||
/** The weekly copy runs every Sunday; past nine days one Sunday was missed. */
|
||||
export const WEEKLY_COPY_ATTENTION_AFTER_DAYS = 9;
|
||||
/** Past fourteen days two Sundays produced nothing for some account. */
|
||||
export const WEEKLY_COPY_FAILING_AFTER_DAYS = 14;
|
||||
/**
|
||||
* How long after the first admin exists the off-host bucket and the recovery
|
||||
* kit may stay unset before the admins are pushed about it.
|
||||
*/
|
||||
export const SETUP_GRACE_DAYS = 7;
|
||||
|
||||
export const READINESS_SIGNALS = [
|
||||
"database_dump",
|
||||
"weekly_copy",
|
||||
"offhost_copies",
|
||||
"restore_drill",
|
||||
"offhost_setup",
|
||||
"recovery_kit",
|
||||
] as const;
|
||||
export type ReadinessSignalId = (typeof READINESS_SIGNALS)[number];
|
||||
|
||||
export const READINESS_STATES = [
|
||||
"ok",
|
||||
"attention",
|
||||
"failing",
|
||||
"inactive",
|
||||
] as const;
|
||||
export type ReadinessState = (typeof READINESS_STATES)[number];
|
||||
|
||||
/** Where each signal is fixed. Every path is an admin page. */
|
||||
export const READINESS_FIX_PATHS: Record<ReadinessSignalId, string> = {
|
||||
database_dump: "/admin/backups",
|
||||
weekly_copy: "/admin/backups",
|
||||
offhost_copies: "/admin/backups",
|
||||
restore_drill: "/admin/backups",
|
||||
offhost_setup: "/admin/backups",
|
||||
recovery_kit: "/admin/encryption",
|
||||
};
|
||||
|
||||
export interface ReadinessSignal {
|
||||
id: ReadinessSignalId;
|
||||
state: ReadinessState;
|
||||
/**
|
||||
* Why, as a closed code per signal. The tile and the push render it through
|
||||
* `admin.recoveryReadiness.<signal>.<reason>`, so it never carries prose.
|
||||
*/
|
||||
reason: string;
|
||||
/** The instant the signal is about (the newest dump, the last drill). */
|
||||
at: string | null;
|
||||
/** Values the sentence for `reason` interpolates. Never key material. */
|
||||
params: Record<string, string | number>;
|
||||
/**
|
||||
* An operator-facing line from the source (a dump's or a drill's failure
|
||||
* text), already redacted where it was written. Shown in the tile only.
|
||||
*/
|
||||
detail: string | null;
|
||||
fixPath: string;
|
||||
}
|
||||
|
||||
export interface RecoveryReadiness {
|
||||
/** The worst state of the signals that apply; `ok` when none do. */
|
||||
state: Exclude<ReadinessState, "inactive">;
|
||||
evaluatedAt: string;
|
||||
signals: ReadinessSignal[];
|
||||
}
|
||||
|
||||
/** What the evaluation reads, gathered by `readiness.ts`. */
|
||||
export interface ReadinessInputs {
|
||||
now: Date;
|
||||
/** When the first admin account was created; null on an empty install. */
|
||||
firstAdminAt: Date | null;
|
||||
/**
|
||||
* `null` while no dump is scheduled by this build; `"unreadable"` when the
|
||||
* dump module threw while reporting, which is itself a failure to report.
|
||||
*/
|
||||
dump: DatabaseBackupHealth | null | "unreadable";
|
||||
/**
|
||||
* Since when the evaluation has seen dumps switched on. A dump that has
|
||||
* never been attempted is judged against this, so an upgrade that turns
|
||||
* dumps on is not failing before the first run had a chance.
|
||||
*/
|
||||
dumpWatchedSince: Date | null;
|
||||
weekly: {
|
||||
/** One entry per account: when it was created and its newest copy. */
|
||||
accounts: ReadonlyArray<{ createdAt: Date; lastCopyAt: Date | null }>;
|
||||
};
|
||||
offhost: {
|
||||
configured: boolean;
|
||||
accounts: ReadonlyArray<{
|
||||
lastAttemptAt: Date | null;
|
||||
lastSuccessAt: Date | null;
|
||||
}>;
|
||||
};
|
||||
drill: {
|
||||
lastRunAt: Date;
|
||||
lastSuccessAt: Date | null;
|
||||
lastFailureAt: Date | null;
|
||||
lastFailureReason: string | null;
|
||||
accountsChecked: number;
|
||||
accountsFailed: number;
|
||||
} | null;
|
||||
kit: {
|
||||
/** A confirmation exists at all. */
|
||||
confirmedAt: Date | null;
|
||||
/** Keys the kit lists that the confirmation does not cover. */
|
||||
unconfirmed: ReadonlyArray<Pick<KitEntry, "kind" | "id">>;
|
||||
};
|
||||
}
|
||||
|
||||
function signal(
|
||||
id: ReadinessSignalId,
|
||||
state: ReadinessState,
|
||||
reason: string,
|
||||
at: Date | null,
|
||||
params: Record<string, string | number> = {},
|
||||
detail: string | null = null,
|
||||
): ReadinessSignal {
|
||||
return {
|
||||
id,
|
||||
state,
|
||||
reason,
|
||||
at: at?.toISOString() ?? null,
|
||||
params,
|
||||
detail,
|
||||
fixPath: READINESS_FIX_PATHS[id],
|
||||
};
|
||||
}
|
||||
|
||||
const wholeHours = (ms: number) => Math.max(0, Math.floor(ms / HOUR_MS));
|
||||
const wholeDays = (ms: number) => Math.max(0, Math.floor(ms / DAY_MS));
|
||||
|
||||
function setupGraceEndsAt(firstAdminAt: Date | null): Date | null {
|
||||
return firstAdminAt
|
||||
? new Date(firstAdminAt.getTime() + SETUP_GRACE_DAYS * DAY_MS)
|
||||
: null;
|
||||
}
|
||||
|
||||
export function evaluateDatabaseDump(
|
||||
input: Pick<ReadinessInputs, "now" | "dump" | "dumpWatchedSince">,
|
||||
): ReadinessSignal {
|
||||
const { dump, now } = input;
|
||||
const id = "database_dump";
|
||||
if (dump === null) return signal(id, "inactive", "not_available", null);
|
||||
if (dump === "unreadable") return signal(id, "failing", "unreadable", null);
|
||||
if (!dump.enabled) return signal(id, "inactive", "disabled", null);
|
||||
const lastGoodAt = dump.lastGood ? new Date(dump.lastGood.at) : null;
|
||||
const lastFailureAt = dump.lastFailure ? new Date(dump.lastFailure.at) : null;
|
||||
const failureReason = dump.lastFailure?.error ?? null;
|
||||
// A directory this process cannot write is a valid setup when the dumps go
|
||||
// to the bucket; with no bucket either, no dump can be kept anywhere.
|
||||
if (!dump.localDirWritable && !dump.offhostConfigured) {
|
||||
return signal(id, "failing", "nowhere_to_write", lastGoodAt, {
|
||||
directory: dump.directory,
|
||||
});
|
||||
}
|
||||
const limitMs = DUMP_FAILING_AFTER_HOURS * HOUR_MS;
|
||||
const failedSinceGood =
|
||||
lastFailureAt !== null &&
|
||||
(lastGoodAt === null || lastFailureAt > lastGoodAt);
|
||||
if (lastGoodAt === null) {
|
||||
if (failedSinceGood) {
|
||||
return signal(
|
||||
id,
|
||||
"failing",
|
||||
"never_succeeded",
|
||||
lastFailureAt,
|
||||
{},
|
||||
failureReason,
|
||||
);
|
||||
}
|
||||
const watched = input.dumpWatchedSince ?? now;
|
||||
return now.getTime() - watched.getTime() > limitMs
|
||||
? signal(id, "failing", "never_ran", null, {
|
||||
hours: DUMP_FAILING_AFTER_HOURS,
|
||||
})
|
||||
: signal(id, "ok", "first_pending", null);
|
||||
}
|
||||
const ageMs = now.getTime() - lastGoodAt.getTime();
|
||||
if (ageMs > limitMs) {
|
||||
return signal(
|
||||
id,
|
||||
"failing",
|
||||
"stale",
|
||||
lastGoodAt,
|
||||
{ hours: wholeHours(ageMs) },
|
||||
failedSinceGood ? failureReason : null,
|
||||
);
|
||||
}
|
||||
if (failedSinceGood) {
|
||||
return signal(
|
||||
id,
|
||||
"attention",
|
||||
"last_failed",
|
||||
lastGoodAt,
|
||||
{},
|
||||
failureReason,
|
||||
);
|
||||
}
|
||||
return signal(id, "ok", "fresh", lastGoodAt);
|
||||
}
|
||||
|
||||
/**
|
||||
* The worst account decides. An account without any copy ages from its
|
||||
* creation, so a new account is not overdue before its first Sunday, and one
|
||||
* the pass keeps skipping becomes overdue like a stale copy would.
|
||||
*/
|
||||
export function evaluateWeeklyCopy(
|
||||
input: Pick<ReadinessInputs, "now" | "weekly">,
|
||||
): ReadinessSignal {
|
||||
const id = "weekly_copy";
|
||||
const { now } = input;
|
||||
if (input.weekly.accounts.length === 0) {
|
||||
return signal(id, "inactive", "no_accounts", null);
|
||||
}
|
||||
let worstMs = -Infinity;
|
||||
let worst: { createdAt: Date; lastCopyAt: Date | null } | null = null;
|
||||
let overdue = 0;
|
||||
for (const account of input.weekly.accounts) {
|
||||
const ref = account.lastCopyAt ?? account.createdAt;
|
||||
const ageMs = now.getTime() - ref.getTime();
|
||||
if (ageMs > WEEKLY_COPY_ATTENTION_AFTER_DAYS * DAY_MS) overdue += 1;
|
||||
if (ageMs > worstMs) {
|
||||
worstMs = ageMs;
|
||||
worst = account;
|
||||
}
|
||||
}
|
||||
const days = wholeDays(worstMs);
|
||||
const at = worst?.lastCopyAt ?? null;
|
||||
const params = { days, accounts: overdue };
|
||||
if (worstMs > WEEKLY_COPY_FAILING_AFTER_DAYS * DAY_MS) {
|
||||
return signal(id, "failing", at ? "stale" : "never", at, params);
|
||||
}
|
||||
if (worstMs > WEEKLY_COPY_ATTENTION_AFTER_DAYS * DAY_MS) {
|
||||
return signal(id, "attention", at ? "stale" : "never", at, params);
|
||||
}
|
||||
// The oldest of the newest copies: the one a restore would be furthest
|
||||
// behind with. Null only while no account has been copied yet.
|
||||
const oldestCopy = input.weekly.accounts
|
||||
.map((a) => a.lastCopyAt)
|
||||
.filter((d): d is Date => d !== null)
|
||||
.reduce<Date | null>((min, d) => (min === null || d < min ? d : min), null);
|
||||
return signal(id, "ok", oldestCopy ? "fresh" : "first_pending", oldestCopy);
|
||||
}
|
||||
|
||||
/**
|
||||
* Any account whose newest off-host object is two schedule periods old (the
|
||||
* freshness module's `stale`, which carries the DST and long-night grace), or
|
||||
* that a run reached and never put an object there.
|
||||
*/
|
||||
export function evaluateOffhostCopies(
|
||||
input: Pick<ReadinessInputs, "now" | "offhost">,
|
||||
): ReadinessSignal {
|
||||
const id = "offhost_copies";
|
||||
if (!input.offhost.configured) {
|
||||
return signal(id, "inactive", "not_configured", null);
|
||||
}
|
||||
const verdicts: Array<{
|
||||
freshness: OffhostBackupFreshness;
|
||||
lastSuccessAt: Date | null;
|
||||
}> = input.offhost.accounts.map((a) => ({
|
||||
freshness: classifyOffhostBackup({
|
||||
lastAttemptAt: a.lastAttemptAt,
|
||||
lastSuccessAt: a.lastSuccessAt,
|
||||
now: input.now,
|
||||
}).freshness,
|
||||
lastSuccessAt: a.lastSuccessAt,
|
||||
}));
|
||||
const oldest = (rows: typeof verdicts) =>
|
||||
rows
|
||||
.map((r) => r.lastSuccessAt)
|
||||
.filter((d): d is Date => d !== null)
|
||||
.reduce<Date | null>(
|
||||
(min, d) => (min === null || d < min ? d : min),
|
||||
null,
|
||||
);
|
||||
const behind = verdicts.filter(
|
||||
(v) => v.freshness === "stale" || v.freshness === "never",
|
||||
);
|
||||
if (behind.length > 0) {
|
||||
return signal(id, "attention", "stale", oldest(behind), {
|
||||
accounts: behind.length,
|
||||
});
|
||||
}
|
||||
const seen = verdicts.filter((v) => v.freshness !== "unknown");
|
||||
if (seen.length === 0) return signal(id, "ok", "first_pending", null);
|
||||
return signal(id, "ok", "fresh", oldest(seen));
|
||||
}
|
||||
|
||||
/** A failed drill stays the verdict until a later drill passes. */
|
||||
export function evaluateRestoreDrill(
|
||||
input: Pick<ReadinessInputs, "offhost" | "drill">,
|
||||
): ReadinessSignal {
|
||||
const id = "restore_drill";
|
||||
const { drill } = input;
|
||||
if (!input.offhost.configured) {
|
||||
return signal(id, "inactive", "not_configured", null);
|
||||
}
|
||||
if (drill === null) return signal(id, "ok", "not_run_yet", null);
|
||||
const failing =
|
||||
drill.lastFailureAt !== null &&
|
||||
(drill.lastSuccessAt === null || drill.lastFailureAt > drill.lastSuccessAt);
|
||||
if (failing) {
|
||||
return signal(
|
||||
id,
|
||||
"failing",
|
||||
// A drill that checked nothing stopped before any object: the bucket
|
||||
// could not be listed, or held nothing to check.
|
||||
drill.accountsChecked === 0 ? "failed_run" : "failed",
|
||||
drill.lastFailureAt,
|
||||
{ failed: drill.accountsFailed, checked: drill.accountsChecked },
|
||||
drill.lastFailureReason,
|
||||
);
|
||||
}
|
||||
return signal(id, "ok", "passed", drill.lastSuccessAt ?? drill.lastRunAt, {
|
||||
checked: drill.accountsChecked,
|
||||
});
|
||||
}
|
||||
|
||||
export function evaluateOffhostSetup(
|
||||
input: Pick<ReadinessInputs, "now" | "firstAdminAt" | "offhost">,
|
||||
): ReadinessSignal {
|
||||
const id = "offhost_setup";
|
||||
if (input.offhost.configured) return signal(id, "ok", "configured", null);
|
||||
const graceEnds = setupGraceEndsAt(input.firstAdminAt);
|
||||
if (graceEnds === null) return signal(id, "inactive", "no_admin", null);
|
||||
return input.now > graceEnds
|
||||
? signal(id, "attention", "not_configured", null)
|
||||
: signal(id, "ok", "not_configured_grace", graceEnds);
|
||||
}
|
||||
|
||||
export function evaluateRecoveryKit(
|
||||
input: Pick<ReadinessInputs, "now" | "firstAdminAt" | "kit">,
|
||||
): ReadinessSignal {
|
||||
const id = "recovery_kit";
|
||||
const { kit } = input;
|
||||
const keys = kit.unconfirmed.map((k) => k.id).join(", ");
|
||||
if (kit.unconfirmed.length === 0 && kit.confirmedAt !== null) {
|
||||
return signal(id, "ok", "confirmed", kit.confirmedAt);
|
||||
}
|
||||
if (kit.confirmedAt !== null) {
|
||||
// Confirmed once, and a key it needs is not in that confirmation: a
|
||||
// rotation, a new backup key, a re-keyed install. No grace: the keys are
|
||||
// already in use.
|
||||
return signal(id, "attention", "out_of_date", kit.confirmedAt, {
|
||||
keys,
|
||||
count: kit.unconfirmed.length,
|
||||
});
|
||||
}
|
||||
const graceEnds = setupGraceEndsAt(input.firstAdminAt);
|
||||
if (graceEnds === null) return signal(id, "inactive", "no_admin", null);
|
||||
return input.now > graceEnds
|
||||
? signal(id, "attention", "not_confirmed", null, {
|
||||
keys,
|
||||
count: kit.unconfirmed.length,
|
||||
})
|
||||
: signal(id, "ok", "not_confirmed_grace", graceEnds, {
|
||||
keys,
|
||||
count: kit.unconfirmed.length,
|
||||
});
|
||||
}
|
||||
|
||||
const SEVERITY: Record<Exclude<ReadinessState, "inactive">, number> = {
|
||||
ok: 0,
|
||||
attention: 1,
|
||||
failing: 2,
|
||||
};
|
||||
|
||||
export function evaluateRecoveryReadiness(
|
||||
inputs: ReadinessInputs,
|
||||
): RecoveryReadiness {
|
||||
const signals = [
|
||||
evaluateDatabaseDump(inputs),
|
||||
evaluateWeeklyCopy(inputs),
|
||||
evaluateOffhostCopies(inputs),
|
||||
evaluateRestoreDrill(inputs),
|
||||
evaluateOffhostSetup(inputs),
|
||||
evaluateRecoveryKit(inputs),
|
||||
];
|
||||
const state = signals.reduce<Exclude<ReadinessState, "inactive">>(
|
||||
(worst, s) =>
|
||||
s.state !== "inactive" && SEVERITY[s.state] > SEVERITY[worst]
|
||||
? s.state
|
||||
: worst,
|
||||
"ok",
|
||||
);
|
||||
return { state, evaluatedAt: inputs.now.toISOString(), signals };
|
||||
}
|
||||
@@ -0,0 +1,205 @@
|
||||
/**
|
||||
* Push recovery readiness to every admin: once when a signal turns to
|
||||
* attention or failing, once when it is back to ok, and a reminder at most
|
||||
* once a day while it stays failing. Nothing else.
|
||||
*
|
||||
* The memory is one row per signal (`RecoveryReadinessNotice`): the state the
|
||||
* admins were last told about and when. A notice is claimed with a
|
||||
* compare-and-set on both columns before it is sent, so two workers that
|
||||
* evaluate in the same minute send it once, and a crash between the claim and
|
||||
* the send loses a notice rather than repeating it every hour.
|
||||
*
|
||||
* Delivery goes through the ordinary localised path, as the deploy-failure
|
||||
* alert does: every ADMIN, in their own locale, on the channels they enabled
|
||||
* for `BACKUP_HEALTH`, with the dispatcher's cooldowns and hard-reject rules.
|
||||
* An admin who switched the event off on a channel is not reached there.
|
||||
*/
|
||||
import { prisma } from "@/lib/db";
|
||||
import { dispatchLocalisedNotification } from "@/lib/notifications/dispatch-localised";
|
||||
import {
|
||||
READINESS_SIGNALS,
|
||||
type ReadinessSignal,
|
||||
type ReadinessSignalId,
|
||||
type ReadinessState,
|
||||
type RecoveryReadiness,
|
||||
} from "@/lib/recovery/readiness-model";
|
||||
|
||||
/** A failing signal is reminded at most this often. */
|
||||
export const FAILING_REMINDER_HOURS = 24;
|
||||
|
||||
type NotifiedState = Exclude<ReadinessState, "inactive">;
|
||||
|
||||
export interface StoredNotice {
|
||||
notifiedState: NotifiedState | null;
|
||||
notifiedAt: Date | null;
|
||||
}
|
||||
|
||||
export type NoticeKind = "problem" | "reminder" | "recovered";
|
||||
|
||||
export interface NoticeDecision {
|
||||
/** What to send, or null for nothing. */
|
||||
send: NoticeKind | null;
|
||||
/** What to store, or null to leave the row as it is. */
|
||||
next: StoredNotice | null;
|
||||
}
|
||||
|
||||
const NOTHING: NoticeDecision = { send: null, next: null };
|
||||
|
||||
/**
|
||||
* Pure: what one evaluation of one signal sends and stores.
|
||||
*
|
||||
* - attention / failing, different from what was last announced: send it.
|
||||
* - failing, as announced, and the last notice is a day old: remind.
|
||||
* - ok after attention or failing: announce the recovery.
|
||||
* - inactive: forget silently. A signal that stops applying (the bucket was
|
||||
* unset, so there is no drill to judge) did not recover, and saying so
|
||||
* would be untrue; if it applies again and is still broken, it is news.
|
||||
*/
|
||||
export function decideNotice(
|
||||
stored: StoredNotice | null,
|
||||
current: ReadinessState,
|
||||
now: Date,
|
||||
): NoticeDecision {
|
||||
const last = stored?.notifiedState ?? null;
|
||||
if (current === "inactive") {
|
||||
return last === null
|
||||
? NOTHING
|
||||
: { send: null, next: { notifiedState: null, notifiedAt: null } };
|
||||
}
|
||||
if (current === "ok") {
|
||||
return last === "attention" || last === "failing"
|
||||
? { send: "recovered", next: { notifiedState: "ok", notifiedAt: now } }
|
||||
: NOTHING;
|
||||
}
|
||||
if (last !== current) {
|
||||
return {
|
||||
send: "problem",
|
||||
next: { notifiedState: current, notifiedAt: now },
|
||||
};
|
||||
}
|
||||
const lastAt = stored?.notifiedAt ?? null;
|
||||
if (
|
||||
current === "failing" &&
|
||||
(lastAt === null ||
|
||||
now.getTime() - lastAt.getTime() >= FAILING_REMINDER_HOURS * 3_600_000)
|
||||
) {
|
||||
return {
|
||||
send: "reminder",
|
||||
next: { notifiedState: current, notifiedAt: now },
|
||||
};
|
||||
}
|
||||
return NOTHING;
|
||||
}
|
||||
|
||||
/** The title of a notice, by what it says. */
|
||||
function titleKey(kind: NoticeKind, state: ReadinessState): string {
|
||||
if (kind === "recovered") {
|
||||
return "notifications.recoveryReadiness.titleRecovered";
|
||||
}
|
||||
if (kind === "reminder") {
|
||||
return "notifications.recoveryReadiness.titleReminder";
|
||||
}
|
||||
return state === "failing"
|
||||
? "notifications.recoveryReadiness.titleFailing"
|
||||
: "notifications.recoveryReadiness.titleAttention";
|
||||
}
|
||||
|
||||
/**
|
||||
* The body: the same sentence the tile shows for the signal's reason, or the
|
||||
* signal's own recovery line.
|
||||
*/
|
||||
function messageKey(kind: NoticeKind, signal: ReadinessSignal): string {
|
||||
return kind === "recovered"
|
||||
? `admin.recoveryReadiness.recovered.${signal.id}`
|
||||
: `admin.recoveryReadiness.reason.${signal.id}.${signal.reason}`;
|
||||
}
|
||||
|
||||
export interface NotifyOutcome {
|
||||
/** Notices claimed and sent, by signal. */
|
||||
sent: Array<{ signal: ReadinessSignalId; kind: NoticeKind }>;
|
||||
/** Admins each notice went to. */
|
||||
admins: number;
|
||||
}
|
||||
|
||||
function isNotifiedState(value: string | null): value is NotifiedState {
|
||||
return value === "ok" || value === "attention" || value === "failing";
|
||||
}
|
||||
|
||||
/**
|
||||
* Compare a fresh evaluation against what the admins were last told, claim
|
||||
* every notice that is due, and send it.
|
||||
*/
|
||||
export async function notifyReadinessChanges(
|
||||
readiness: RecoveryReadiness,
|
||||
now: Date = new Date(readiness.evaluatedAt),
|
||||
): Promise<NotifyOutcome> {
|
||||
await prisma.recoveryReadinessNotice.createMany({
|
||||
data: READINESS_SIGNALS.map((signal) => ({ signal })),
|
||||
skipDuplicates: true,
|
||||
});
|
||||
const rows = await prisma.recoveryReadinessNotice.findMany();
|
||||
const byId = new Map(rows.map((row) => [row.signal, row]));
|
||||
const admins = await prisma.user.findMany({
|
||||
where: { role: "ADMIN" },
|
||||
select: { id: true },
|
||||
});
|
||||
|
||||
const outcome: NotifyOutcome = { sent: [], admins: admins.length };
|
||||
for (const signal of readiness.signals) {
|
||||
const row = byId.get(signal.id);
|
||||
const applies = signal.state !== "inactive";
|
||||
// Since when the signal applies: set once, cleared when it stops.
|
||||
if (row && applies !== (row.watchedSince !== null)) {
|
||||
await prisma.recoveryReadinessNotice.updateMany({
|
||||
where: { signal: signal.id, watchedSince: row.watchedSince },
|
||||
data: { watchedSince: applies ? now : null },
|
||||
});
|
||||
}
|
||||
|
||||
const stored: StoredNotice | null = row
|
||||
? {
|
||||
notifiedState: isNotifiedState(row.notifiedState)
|
||||
? row.notifiedState
|
||||
: null,
|
||||
notifiedAt: row.notifiedAt,
|
||||
}
|
||||
: null;
|
||||
const decision = decideNotice(stored, signal.state, now);
|
||||
// With no admin yet there is nobody to tell, so nothing is claimed: the
|
||||
// first admin hears about a problem that is still standing.
|
||||
if (!decision.next || admins.length === 0) continue;
|
||||
|
||||
// Claim: only the evaluation that moves the row from what it read sends.
|
||||
const claimed = await prisma.recoveryReadinessNotice.updateMany({
|
||||
where: {
|
||||
signal: signal.id,
|
||||
notifiedState: row?.notifiedState ?? null,
|
||||
notifiedAt: row?.notifiedAt ?? null,
|
||||
},
|
||||
data: {
|
||||
notifiedState: decision.next.notifiedState,
|
||||
notifiedAt: decision.next.notifiedAt,
|
||||
},
|
||||
});
|
||||
if (claimed.count !== 1 || decision.send === null) continue;
|
||||
|
||||
for (const admin of admins) {
|
||||
await dispatchLocalisedNotification({
|
||||
userId: admin.id,
|
||||
eventType: "BACKUP_HEALTH",
|
||||
titleKey: titleKey(decision.send, signal.state),
|
||||
messageKey: messageKey(decision.send, signal),
|
||||
params: signal.params,
|
||||
metadata: {
|
||||
source: "recovery-readiness",
|
||||
url: signal.fixPath,
|
||||
signal: signal.id,
|
||||
state: signal.state,
|
||||
reason: signal.reason,
|
||||
},
|
||||
});
|
||||
}
|
||||
outcome.sent.push({ signal: signal.id, kind: decision.send });
|
||||
}
|
||||
return outcome;
|
||||
}
|
||||
@@ -0,0 +1,117 @@
|
||||
/**
|
||||
* Read every fact the recovery readiness model judges, and judge it.
|
||||
*
|
||||
* One reader for both consumers: the admin tile (`GET
|
||||
* /api/admin/recovery-readiness`) and the hourly evaluation that pushes state
|
||||
* changes to the admins. Each fact comes from where its own surface already
|
||||
* reads it, so the tile can never disagree with the backups page about a copy:
|
||||
*
|
||||
* - the weekly copy: the `WEEKLY_AUTO` row per account, which never ages out
|
||||
* of any retention window (pg-boss forgets a run after seven days);
|
||||
* - the off-host copies: the per-account ledger the nightly run writes, and
|
||||
* the same "configured" rule the backups page uses (the environment, or a
|
||||
* ledger row as proof that an upload happened);
|
||||
* - the drill: its persisted last outcome, with no time window;
|
||||
* - the kit: the same assessment the kit card shows;
|
||||
* - the dump: whatever the dump module reports, through a provider.
|
||||
*/
|
||||
import { prisma } from "@/lib/db";
|
||||
import { offhostBackupConfigured } from "@/lib/jobs/offhost-backup";
|
||||
import {
|
||||
readDatabaseBackupHealth,
|
||||
type DatabaseBackupHealthProvider,
|
||||
} from "@/lib/recovery/database-dump-health";
|
||||
import { readRecoveryKit } from "@/lib/recovery/kit";
|
||||
import {
|
||||
evaluateRecoveryReadiness,
|
||||
type ReadinessInputs,
|
||||
type RecoveryReadiness,
|
||||
} from "@/lib/recovery/readiness-model";
|
||||
|
||||
/** The type the weekly worker writes; anything else was uploaded by hand. */
|
||||
const SCHEDULED_BACKUP_TYPE = "WEEKLY_AUTO";
|
||||
|
||||
export interface ReadRecoveryReadinessOptions {
|
||||
now?: Date;
|
||||
dumpHealth?: DatabaseBackupHealthProvider;
|
||||
}
|
||||
|
||||
export async function readReadinessInputs(
|
||||
options: ReadRecoveryReadinessOptions = {},
|
||||
): Promise<ReadinessInputs> {
|
||||
const now = options.now ?? new Date();
|
||||
const dumpHealth = options.dumpHealth ?? readDatabaseBackupHealth;
|
||||
const [firstAdmin, accounts, drill, dumpNotice, dump, kit] =
|
||||
await Promise.all([
|
||||
prisma.user.findFirst({
|
||||
where: { role: "ADMIN" },
|
||||
orderBy: { createdAt: "asc" },
|
||||
select: { createdAt: true },
|
||||
}),
|
||||
prisma.user.findMany({
|
||||
select: {
|
||||
createdAt: true,
|
||||
dataBackups: {
|
||||
where: { type: SCHEDULED_BACKUP_TYPE },
|
||||
select: { createdAt: true },
|
||||
},
|
||||
offhostBackupState: {
|
||||
select: { lastAttemptAt: true, lastSuccessAt: true },
|
||||
},
|
||||
},
|
||||
}),
|
||||
prisma.restoreDrillOutcome.findUnique({ where: { id: "singleton" } }),
|
||||
prisma.recoveryReadinessNotice.findUnique({
|
||||
where: { signal: "database_dump" },
|
||||
select: { watchedSince: true },
|
||||
}),
|
||||
dumpHealth(now).catch(() => "unreadable" as const),
|
||||
readRecoveryKit(now),
|
||||
]);
|
||||
|
||||
// Evidence beats this process's environment, as on the backups page: a
|
||||
// ledger row is proof an upload happened, whichever process reads it.
|
||||
const configured =
|
||||
offhostBackupConfigured() ||
|
||||
accounts.some((account) => account.offhostBackupState !== null);
|
||||
|
||||
return {
|
||||
now,
|
||||
firstAdminAt: firstAdmin?.createdAt ?? null,
|
||||
dump,
|
||||
dumpWatchedSince: dumpNotice?.watchedSince ?? null,
|
||||
weekly: {
|
||||
accounts: accounts.map((account) => ({
|
||||
createdAt: account.createdAt,
|
||||
lastCopyAt: account.dataBackups[0]?.createdAt ?? null,
|
||||
})),
|
||||
},
|
||||
offhost: {
|
||||
configured,
|
||||
accounts: accounts.map((account) => ({
|
||||
lastAttemptAt: account.offhostBackupState?.lastAttemptAt ?? null,
|
||||
lastSuccessAt: account.offhostBackupState?.lastSuccessAt ?? null,
|
||||
})),
|
||||
},
|
||||
drill: drill
|
||||
? {
|
||||
lastRunAt: drill.lastRunAt,
|
||||
lastSuccessAt: drill.lastSuccessAt,
|
||||
lastFailureAt: drill.lastFailureAt,
|
||||
lastFailureReason: drill.lastFailureReason,
|
||||
accountsChecked: drill.accountsChecked,
|
||||
accountsFailed: drill.accountsFailed,
|
||||
}
|
||||
: null,
|
||||
kit: {
|
||||
confirmedAt: kit.confirmedAt ? new Date(kit.confirmedAt) : null,
|
||||
unconfirmed: kit.unconfirmed,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export async function readRecoveryReadiness(
|
||||
options: ReadRecoveryReadinessOptions = {},
|
||||
): Promise<RecoveryReadiness> {
|
||||
return evaluateRecoveryReadiness(await readReadinessInputs(options));
|
||||
}
|
||||
@@ -0,0 +1,295 @@
|
||||
/**
|
||||
* Recovery readiness, end to end against Postgres: a weekly copy that stopped
|
||||
* reaches every admin exactly once per state change, through the real
|
||||
* readiness reader, the real notice ledger, the real dispatcher with its
|
||||
* preference lookup and ledger, and the real APNs sender (vendor client
|
||||
* stubbed, the shape `delegated-intake-notification.test.ts` uses).
|
||||
*
|
||||
* The facts:
|
||||
*
|
||||
* 1. A stale copy becomes one `BACKUP_HEALTH` delivery per admin; the next
|
||||
* hourly runs at the same state add none.
|
||||
* 2. Escalating to failing adds one; a reminder comes a day later and not
|
||||
* sooner; the recovery adds one; ok after that adds none.
|
||||
* 3. An admin who switched the event off on the channel gets nothing, and
|
||||
* the notice is still spent, so switching it back on later does not
|
||||
* replay an old alarm.
|
||||
* 4. The hourly job runs the same path from its handler.
|
||||
*
|
||||
* The ledger write is fire-and-forget in `recordPushAttempt`, so counts are
|
||||
* polled up to the expected number and then asserted exactly.
|
||||
*/
|
||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
|
||||
process.env.ENCRYPTION_KEY ??=
|
||||
"0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef";
|
||||
|
||||
const apnsSendMock = vi.fn();
|
||||
|
||||
vi.mock("@parse/node-apn", () => {
|
||||
class Provider {
|
||||
constructor(_opts: unknown) {
|
||||
void _opts;
|
||||
}
|
||||
send(...args: unknown[]) {
|
||||
return apnsSendMock(...args);
|
||||
}
|
||||
shutdown() {}
|
||||
}
|
||||
class Notification {
|
||||
public topic = "";
|
||||
public alert: unknown;
|
||||
public sound: string | undefined;
|
||||
public badge: number | undefined;
|
||||
public threadId: string | undefined;
|
||||
public collapseId: string | undefined;
|
||||
public payload: unknown;
|
||||
}
|
||||
return { default: { Provider, Notification }, Provider, Notification };
|
||||
});
|
||||
|
||||
vi.mock("@/lib/db-compat", () => ({
|
||||
ensureDbCompatibility: vi.fn().mockResolvedValue(undefined),
|
||||
}));
|
||||
|
||||
import { getPrismaClient, truncateAllTables } from "./setup";
|
||||
|
||||
const ADMIN_ID = "user-readiness-admin";
|
||||
const SECOND_ADMIN_ID = "user-readiness-admin-2";
|
||||
const MEMBER_ID = "user-readiness-member";
|
||||
const DAY = 86_400_000;
|
||||
const HOUR = 3_600_000;
|
||||
|
||||
const TEST_EC_PEM_LINES = [
|
||||
"-----BEGIN PRIVATE KEY-----",
|
||||
"MIGTAgEAMBMGByqGSM49AgEGCCqGSM49AwEHBHkwdwIBAQQgLOXP3Exjr5L5tamN",
|
||||
"pTxck85Iaum80PdRlWDpc/ezviOgCgYIKoZIzj0DAQehRANCAAT1x8nKRb8KshQU",
|
||||
"1aPieSCqOY6ilgC959umaFSlhfav8eZ91UHP/xond9aMoZcuQ7lJG/Rsj70SWMvZ",
|
||||
"bw81BG89",
|
||||
"-----END PRIVATE KEY-----",
|
||||
];
|
||||
|
||||
const APNS_ENV = {
|
||||
APNS_KEY_ID: "ABCDE12345",
|
||||
APNS_TEAM_ID: "TEAM123456",
|
||||
APNS_BUNDLE_ID: "test.healthlog.ios",
|
||||
APNS_KEY: TEST_EC_PEM_LINES.join("\\n"),
|
||||
};
|
||||
|
||||
async function seedApnsChannel(userId: string): Promise<string> {
|
||||
const { encrypt } = await import("@/lib/crypto");
|
||||
const channel = await getPrismaClient().notificationChannel.create({
|
||||
data: { userId, type: "APNS", enabled: true, config: encrypt("{}") },
|
||||
});
|
||||
await getPrismaClient().device.create({
|
||||
data: {
|
||||
userId,
|
||||
platform: "ios",
|
||||
token: `device-${userId}`,
|
||||
bundleId: APNS_ENV.APNS_BUNDLE_ID,
|
||||
apnsToken: `apns-${userId}`,
|
||||
apnsEnvironment: "sandbox",
|
||||
},
|
||||
});
|
||||
return channel.id;
|
||||
}
|
||||
|
||||
async function backupAttempts(
|
||||
userId: string,
|
||||
expected: number,
|
||||
): Promise<number> {
|
||||
const count = () =>
|
||||
getPrismaClient().pushAttempt.count({
|
||||
where: { userId, eventType: "BACKUP_HEALTH" },
|
||||
});
|
||||
for (let attempt = 0; attempt < 40; attempt += 1) {
|
||||
if ((await count()) >= expected) break;
|
||||
await new Promise((r) => setTimeout(r, 25));
|
||||
}
|
||||
// A short settle, so a delivery that should NOT have happened has the same
|
||||
// chance to land as one that should.
|
||||
await new Promise((r) => setTimeout(r, 75));
|
||||
return count();
|
||||
}
|
||||
|
||||
/** Put the member's weekly copy `ageDays` before `now`. */
|
||||
async function weeklyCopyAged(ageDays: number, now: Date): Promise<void> {
|
||||
await getPrismaClient().dataBackup.upsert({
|
||||
where: { userId_type: { userId: MEMBER_ID, type: "WEEKLY_AUTO" } },
|
||||
create: {
|
||||
userId: MEMBER_ID,
|
||||
type: "WEEKLY_AUTO",
|
||||
data: "sealed",
|
||||
createdAt: new Date(now.getTime() - ageDays * DAY),
|
||||
},
|
||||
update: { createdAt: new Date(now.getTime() - ageDays * DAY) },
|
||||
});
|
||||
}
|
||||
|
||||
/** One hourly evaluation at `now`, through the reader and the notifier. */
|
||||
async function evaluateAt(now: Date) {
|
||||
const { readRecoveryReadiness } = await import("@/lib/recovery/readiness");
|
||||
const { notifyReadinessChanges } =
|
||||
await import("@/lib/recovery/readiness-notify");
|
||||
const readiness = await readRecoveryReadiness({ now });
|
||||
await notifyReadinessChanges(readiness, now);
|
||||
return readiness;
|
||||
}
|
||||
|
||||
beforeEach(async () => {
|
||||
vi.clearAllMocks();
|
||||
for (const [k, v] of Object.entries(APNS_ENV)) process.env[k] = v;
|
||||
delete process.env.APNS_PRODUCTION;
|
||||
for (const name of [
|
||||
"BACKUP_S3_ENDPOINT",
|
||||
"BACKUP_S3_BUCKET",
|
||||
"BACKUP_S3_ACCESS_KEY",
|
||||
"BACKUP_S3_SECRET_KEY",
|
||||
"BACKUP_ENCRYPTION_KEY",
|
||||
]) {
|
||||
delete process.env[name];
|
||||
}
|
||||
const apns = await import("@/lib/notifications/senders/apns");
|
||||
apns.resetApnsForTesting();
|
||||
const { __resetDispatchLocaleCacheForTests } =
|
||||
await import("@/lib/notifications/dispatch-localised");
|
||||
__resetDispatchLocaleCacheForTests();
|
||||
|
||||
const prisma = getPrismaClient();
|
||||
await truncateAllTables(prisma);
|
||||
// Admins created "now": the setup grace keeps the off-host and kit signals
|
||||
// quiet, so the weekly copy is the only signal that moves.
|
||||
await prisma.user.createMany({
|
||||
data: [
|
||||
{
|
||||
id: ADMIN_ID,
|
||||
username: "readiness-admin",
|
||||
email: "readiness-admin@example.test",
|
||||
role: "ADMIN",
|
||||
locale: "en",
|
||||
},
|
||||
{
|
||||
id: SECOND_ADMIN_ID,
|
||||
username: "readiness-admin-2",
|
||||
email: "readiness-admin-2@example.test",
|
||||
role: "ADMIN",
|
||||
locale: "de",
|
||||
},
|
||||
{
|
||||
id: MEMBER_ID,
|
||||
username: "readiness-member",
|
||||
email: "readiness-member@example.test",
|
||||
createdAt: new Date(Date.now() - 90 * DAY),
|
||||
},
|
||||
],
|
||||
});
|
||||
// An admin's own account needs a fresh copy too, or it would age from its
|
||||
// creation and move the verdict on its own.
|
||||
const now = new Date();
|
||||
for (const userId of [ADMIN_ID, SECOND_ADMIN_ID]) {
|
||||
await prisma.dataBackup.create({
|
||||
data: { userId, type: "WEEKLY_AUTO", data: "sealed", createdAt: now },
|
||||
});
|
||||
}
|
||||
|
||||
apnsSendMock.mockImplementation(async (note: unknown) => {
|
||||
void note;
|
||||
return { sent: [{ device: "x" }], failed: [] };
|
||||
});
|
||||
});
|
||||
|
||||
describe("recovery readiness push", () => {
|
||||
it("notifies every admin once per state change, reminds daily while failing, and announces the recovery once", async () => {
|
||||
await seedApnsChannel(ADMIN_ID);
|
||||
await seedApnsChannel(SECOND_ADMIN_ID);
|
||||
const t0 = new Date();
|
||||
const at = (hours: number) => new Date(t0.getTime() + hours * HOUR);
|
||||
|
||||
// Ten days: attention.
|
||||
await weeklyCopyAged(10, t0);
|
||||
const first = await evaluateAt(t0);
|
||||
expect(first.signals.find((s) => s.id === "weekly_copy")).toMatchObject({
|
||||
state: "attention",
|
||||
reason: "stale",
|
||||
});
|
||||
expect(first.signals.find((s) => s.id === "recovery_kit")?.state).toBe(
|
||||
"ok",
|
||||
);
|
||||
await evaluateAt(at(1));
|
||||
await evaluateAt(at(2));
|
||||
expect(await backupAttempts(ADMIN_ID, 1)).toBe(1);
|
||||
expect(await backupAttempts(SECOND_ADMIN_ID, 1)).toBe(1);
|
||||
|
||||
// Each admin in their own language.
|
||||
const titles = apnsSendMock.mock.calls.map(
|
||||
([note]) => (note as { alert: { title: string } }).alert.title,
|
||||
);
|
||||
expect(titles).toContain("Backups need attention");
|
||||
expect(titles).toContain("Sicherungen brauchen Aufmerksamkeit");
|
||||
|
||||
// Fifteen days: failing, once.
|
||||
await weeklyCopyAged(15, at(3));
|
||||
await evaluateAt(at(3));
|
||||
await evaluateAt(at(4));
|
||||
expect(await backupAttempts(ADMIN_ID, 2)).toBe(2);
|
||||
|
||||
// Still failing: nothing until a day after the last notice, then one.
|
||||
await evaluateAt(at(3 + 23));
|
||||
expect(await backupAttempts(ADMIN_ID, 3)).toBe(2);
|
||||
await evaluateAt(at(3 + 24));
|
||||
await evaluateAt(at(3 + 25));
|
||||
expect(await backupAttempts(ADMIN_ID, 3)).toBe(3);
|
||||
|
||||
// A fresh copy: one recovery notice, then silence.
|
||||
await weeklyCopyAged(0, at(30));
|
||||
await evaluateAt(at(30));
|
||||
await evaluateAt(at(31));
|
||||
expect(await backupAttempts(ADMIN_ID, 4)).toBe(4);
|
||||
expect(await backupAttempts(SECOND_ADMIN_ID, 4)).toBe(4);
|
||||
|
||||
const notice = await getPrismaClient().recoveryReadinessNotice.findUnique({
|
||||
where: { signal: "weekly_copy" },
|
||||
});
|
||||
expect(notice?.notifiedState).toBe("ok");
|
||||
});
|
||||
|
||||
it("respects an admin's per-channel opt-out and still spends the notice", async () => {
|
||||
const channelId = await seedApnsChannel(ADMIN_ID);
|
||||
await getPrismaClient().notificationPreference.create({
|
||||
data: { channelId, eventType: "BACKUP_HEALTH", enabled: false },
|
||||
});
|
||||
const t0 = new Date();
|
||||
await weeklyCopyAged(10, t0);
|
||||
await evaluateAt(t0);
|
||||
expect(await backupAttempts(ADMIN_ID, 1)).toBe(0);
|
||||
|
||||
await getPrismaClient().notificationPreference.updateMany({
|
||||
where: { channelId },
|
||||
data: { enabled: true },
|
||||
});
|
||||
await evaluateAt(new Date(t0.getTime() + HOUR));
|
||||
expect(await backupAttempts(ADMIN_ID, 1)).toBe(0);
|
||||
const notice = await getPrismaClient().recoveryReadinessNotice.findUnique({
|
||||
where: { signal: "weekly_copy" },
|
||||
});
|
||||
expect(notice?.notifiedState).toBe("attention");
|
||||
});
|
||||
|
||||
it("runs the same path from the hourly job", async () => {
|
||||
await seedApnsChannel(ADMIN_ID);
|
||||
await weeklyCopyAged(10, new Date());
|
||||
const { handleRecoveryReadiness } =
|
||||
await import("@/lib/jobs/recovery-readiness");
|
||||
const outcome = await handleRecoveryReadiness([]);
|
||||
expect(outcome).toMatchObject({
|
||||
ok: true,
|
||||
did: {
|
||||
recovery_readiness_state: "attention",
|
||||
recovery_readiness_notices: 1,
|
||||
recovery_readiness_admins: 2,
|
||||
},
|
||||
});
|
||||
await handleRecoveryReadiness([]);
|
||||
expect(await backupAttempts(ADMIN_ID, 1)).toBe(1);
|
||||
});
|
||||
});
|
||||
Reference in new issue
Block a user