diff --git a/messages/de.json b/messages/de.json index bd57dc6e4..17e7aaab4 100644 --- a/messages/de.json +++ b/messages/de.json @@ -7276,7 +7276,12 @@ "neverDetail": "Für dieses Konto ist nichts angekommen. Zuletzt geprüft {when}.", "unknownDetail": "Noch kein nächtlicher Lauf hat dieses Konto erfasst.", "age": "vor {hours} h", - "scheduleNote": "Der Upload läuft alle {hours} h. Frisch heißt innerhalb eines Laufs, fällig heißt ein Lauf blieb ohne Ergebnis, veraltet heißt zwei." + "scheduleNote": "Der Upload läuft alle {hours} h. Frisch heißt innerhalb eines Laufs, fällig heißt ein Lauf blieb ohne Ergebnis, veraltet heißt zwei.", + "lifecycleConfigured": "Der Bucket löscht Kopien nach {days} Tagen über seine eigene Lifecycle-Regel.", + "lifecycleMissing": "Der Bucket hat keine Lifecycle-Regel, jede nächtliche Kopie bleibt also für immer liegen. Lege eine Regel an, die Objekte nach BACKUP_RETENTION_DAYS löscht.", + "lifecycleUnknown": "Ob der Bucket alte Kopien löscht, ließ sich nicht lesen: Die Zugangsdaten dürfen seine Lifecycle-Regel nicht lesen. Prüfe, dass eine Regel Objekte nach BACKUP_RETENTION_DAYS löscht.", + "pendingDeletions": "Kopien von {count} gelöschten oder geleerten Konten liegen noch im Bucket und werden entfernt.", + "pendingDeletionsFailing": "Kopien von {count} gelöschten oder geleerten Konten liegen noch im Bucket, weil er das Löschen verweigert hat ({error}). Gib den Zugangsdaten DeleteObject; das Löschen wird jede Nacht wiederholt." }, "restoreStarted": "Wiederherstellung gestartet. Der Fortschritt steht über der Liste.", "restoreActive": "Für dieses Konto läuft schon eine Wiederherstellung. Warte, bis sie fertig ist.", @@ -7364,7 +7369,17 @@ "completed": "Abgeschlossen", "failed": "Fehlgeschlagen" }, - "coverageDetail": "Es werden nur Schlüssel-IDs und Anzahlen gezeigt — niemals Schlüsselmaterial." + "coverageDetail": "Es werden nur Schlüssel-IDs und Anzahlen gezeigt — niemals Schlüsselmaterial.", + "backupsTitle": "Schlüssel, die die Sicherungen noch brauchen", + "backupsDescription": "Ein alter Schlüssel bleibt nötig, solange eine Sicherung unten ihn noch aufführt.", + "backupsStoredLine": "Schlüssel {key}: {count} gespeicherte Kopien, die älteste vom {when}", + "backupsUnrecordedLine": "{count} gespeicherte Kopien stammen aus der Zeit, bevor Sicherungen ihre Schlüssel festhielten (die älteste vom {when}). Sie können jeden damals vorhandenen Schlüssel brauchen: Behalte alle alten Schlüssel, bis sie ersetzt oder gelöscht sind.", + "backupsOffhostLine": "Schlüssel {key}: Die letzte externe Kopie, die ihn braucht, kam am {when} in den Bucket; dort gebraucht bis {until}", + "backupsOffhostLineUnknown": "Schlüssel {key}: Die letzte externe Kopie, die ihn braucht, kam am {when} in den Bucket. Wie lange der Bucket Kopien behält, ist unbekannt; behalte den Schlüssel, bis diese Kopie sicher weg ist.", + "backupsNone": "Noch keine Sicherung hat die Schlüssel festgehalten, die sie braucht.", + "backupsStillNeed": "Alle Zeilen sind rotiert, aber Sicherungen brauchen noch {keys}", + "backupsUnrecordedBadge": "Alle Zeilen sind rotiert, aber ältere Sicherungen können noch einen ausgemusterten Schlüssel brauchen", + "backupsDetail": "Die Rotation verschlüsselt jede Zeile neu, aber eine Sicherung behält den Schlüssel, unter dem ihr Inhalt geschrieben wurde, und ihre Wiederherstellung braucht ihn. Behalte einen ausgemusterten Schlüssel in ENCRYPTION_KEYS, bis er unten nicht mehr aufgeführt ist." } }, "carrier": "Netzbetreiber", diff --git a/messages/en.json b/messages/en.json index f1d2fa36c..aa1ae1df6 100644 --- a/messages/en.json +++ b/messages/en.json @@ -7276,7 +7276,12 @@ "neverDetail": "Nothing has landed for this account. Last checked {when}.", "unknownDetail": "No nightly run has recorded this account yet.", "age": "{hours} h ago", - "scheduleNote": "The upload runs every {hours} h. Fresh means inside one run, due means one run produced nothing, stale means two did not." + "scheduleNote": "The upload runs every {hours} h. Fresh means inside one run, due means one run produced nothing, stale means two did not.", + "lifecycleConfigured": "The bucket deletes copies after {days} days by its own lifecycle rule.", + "lifecycleMissing": "The bucket has no lifecycle rule, so every nightly copy is kept forever. Add a rule that expires objects after BACKUP_RETENTION_DAYS.", + "lifecycleUnknown": "Whether the bucket deletes old copies could not be read: the credential may not read its lifecycle rule. Check that a rule expires objects after BACKUP_RETENTION_DAYS.", + "pendingDeletions": "Copies of {count} deleted or wiped accounts are still in the bucket and are being removed.", + "pendingDeletionsFailing": "Copies of {count} deleted or wiped accounts are still in the bucket because it refused to delete them ({error}). Give the credential DeleteObject; the deletion is retried every night." }, "restoreStarted": "Restore started. Its progress shows above the list.", "restoreActive": "A restore of this account is already running. Wait for it to finish.", @@ -7364,7 +7369,17 @@ "completed": "Completed", "failed": "Failed" }, - "coverageDetail": "Only key ids and counts are shown — never key material." + "coverageDetail": "Only key ids and counts are shown — never key material.", + "backupsTitle": "Keys the backups still need", + "backupsDescription": "A retired key stays needed while any backup below still lists it.", + "backupsStoredLine": "Key {key}: {count} stored copies, the oldest from {when}", + "backupsUnrecordedLine": "{count} stored copies were written before backups recorded their keys (the oldest on {when}). They may need any key that existed then: keep every old key until they are replaced or deleted.", + "backupsOffhostLine": "Key {key}: the last off-host copy that needs it went into the bucket on {when}; needed there until {until}", + "backupsOffhostLineUnknown": "Key {key}: the last off-host copy that needs it went into the bucket on {when}. The bucket's expiry is unknown, so keep the key until you know that copy is gone.", + "backupsNone": "No backup has recorded the keys it needs yet.", + "backupsStillNeed": "Every row is rotated, but backups still need {keys}", + "backupsUnrecordedBadge": "Every row is rotated, but older backups may still need a retired key", + "backupsDetail": "Rotation re-encrypts every row, but a backup keeps the key its content was written under, and restoring it needs that key. Keep a retired key in ENCRYPTION_KEYS until nothing below lists it." } }, "carrier": "Carrier", diff --git a/messages/es.json b/messages/es.json index 6381fb29a..f2d6e1242 100644 --- a/messages/es.json +++ b/messages/es.json @@ -7276,7 +7276,12 @@ "neverDetail": "No ha llegado nada para esta cuenta. Última comprobación {when}.", "unknownDetail": "Ninguna ejecución nocturna ha registrado esta cuenta todavía.", "age": "hace {hours} h", - "scheduleNote": "La subida se ejecuta cada {hours} h. Reciente significa dentro de una ejecución, pendiente que una no produjo nada, obsoleta que fueron dos." + "scheduleNote": "La subida se ejecuta cada {hours} h. Reciente significa dentro de una ejecución, pendiente que una no produjo nada, obsoleta que fueron dos.", + "lifecycleConfigured": "El bucket borra las copias a los {days} días con su propia regla de ciclo de vida.", + "lifecycleMissing": "El bucket no tiene regla de ciclo de vida, así que cada copia nocturna se guarda para siempre. Añade una regla que caduque los objetos tras BACKUP_RETENTION_DAYS.", + "lifecycleUnknown": "No se pudo leer si el bucket borra las copias antiguas: la credencial no puede leer su regla de ciclo de vida. Comprueba que una regla caduque los objetos tras BACKUP_RETENTION_DAYS.", + "pendingDeletions": "Aún quedan en el bucket copias de {count} cuentas eliminadas o vaciadas, y se están borrando.", + "pendingDeletionsFailing": "Aún quedan en el bucket copias de {count} cuentas eliminadas o vaciadas porque rechazó borrarlas ({error}). Da a la credencial el permiso DeleteObject; el borrado se reintenta cada noche." }, "restoreStarted": "Restauración iniciada. Su progreso aparece encima de la lista.", "restoreActive": "Ya hay una restauración en curso para esta cuenta. Espera a que termine.", @@ -7364,7 +7369,17 @@ "completed": "Completado", "failed": "Fallido" }, - "coverageDetail": "Solo se muestran los id de clave y los recuentos, nunca el material de la clave." + "coverageDetail": "Solo se muestran los id de clave y los recuentos, nunca el material de la clave.", + "backupsTitle": "Claves que las copias aún necesitan", + "backupsDescription": "Una clave retirada sigue siendo necesaria mientras alguna copia de abajo la indique.", + "backupsStoredLine": "Clave {key}: {count} copias guardadas, la más antigua del {when}", + "backupsUnrecordedLine": "{count} copias guardadas se escribieron antes de que las copias registraran sus claves (la más antigua el {when}). Pueden necesitar cualquier clave que existiera entonces: conserva todas las claves antiguas hasta que se sustituyan o borren.", + "backupsOffhostLine": "Clave {key}: la última copia externa que la necesita entró en el bucket el {when}; necesaria allí hasta el {until}", + "backupsOffhostLineUnknown": "Clave {key}: la última copia externa que la necesita entró en el bucket el {when}. No se sabe cuándo caducan las copias en el bucket, así que conserva la clave hasta saber que esa copia ya no está.", + "backupsNone": "Ninguna copia ha registrado todavía las claves que necesita.", + "backupsStillNeed": "Todas las filas están rotadas, pero las copias aún necesitan {keys}", + "backupsUnrecordedBadge": "Todas las filas están rotadas, pero copias antiguas pueden necesitar aún una clave retirada", + "backupsDetail": "La rotación vuelve a cifrar cada fila, pero una copia conserva la clave con la que se escribió su contenido, y restaurarla necesita esa clave. Mantén una clave retirada en ENCRYPTION_KEYS hasta que no aparezca abajo." } }, "carrier": "Operador", diff --git a/messages/fr.json b/messages/fr.json index 8308f8e1a..afd9e2f1e 100644 --- a/messages/fr.json +++ b/messages/fr.json @@ -7276,7 +7276,12 @@ "neverDetail": "Rien n'est arrivé pour ce compte. Dernière vérification {when}.", "unknownDetail": "Aucun cycle nocturne n'a encore enregistré ce compte.", "age": "il y a {hours} h", - "scheduleNote": "L'envoi tourne toutes les {hours} h. Récente signifie dans un cycle, attendue qu'un cycle n'a rien produit, périmée que deux n'ont rien produit." + "scheduleNote": "L'envoi tourne toutes les {hours} h. Récente signifie dans un cycle, attendue qu'un cycle n'a rien produit, périmée que deux n'ont rien produit.", + "lifecycleConfigured": "Le bucket supprime les copies après {days} jours grâce à sa propre règle de cycle de vie.", + "lifecycleMissing": "Le bucket n'a aucune règle de cycle de vie : chaque copie nocturne est donc conservée indéfiniment. Ajoutez une règle qui fait expirer les objets après BACKUP_RETENTION_DAYS.", + "lifecycleUnknown": "Impossible de savoir si le bucket supprime les anciennes copies : l'identifiant ne peut pas lire sa règle de cycle de vie. Vérifiez qu'une règle fait expirer les objets après BACKUP_RETENTION_DAYS.", + "pendingDeletions": "Des copies de {count} comptes supprimés ou vidés sont encore dans le bucket et sont en cours de suppression.", + "pendingDeletionsFailing": "Des copies de {count} comptes supprimés ou vidés sont encore dans le bucket, qui a refusé de les supprimer ({error}). Accordez DeleteObject à l'identifiant ; la suppression est retentée chaque nuit." }, "restoreStarted": "Restauration lancée. Sa progression s'affiche au-dessus de la liste.", "restoreActive": "Une restauration de ce compte est déjà en cours. Attends qu'elle se termine.", @@ -7364,7 +7369,17 @@ "completed": "Terminé", "failed": "Échoué" }, - "coverageDetail": "Seuls les id de clé et les décomptes sont affichés, jamais le matériel de clé." + "coverageDetail": "Seuls les id de clé et les décomptes sont affichés, jamais le matériel de clé.", + "backupsTitle": "Clés dont les sauvegardes ont encore besoin", + "backupsDescription": "Une clé retirée reste nécessaire tant qu'une sauvegarde ci-dessous la mentionne.", + "backupsStoredLine": "Clé {key} : {count} copies stockées, la plus ancienne du {when}", + "backupsUnrecordedLine": "{count} copies stockées ont été écrites avant que les sauvegardes n'enregistrent leurs clés (la plus ancienne le {when}). Elles peuvent avoir besoin de n'importe quelle clé de l'époque : gardez toutes les anciennes clés jusqu'à leur remplacement ou suppression.", + "backupsOffhostLine": "Clé {key} : la dernière copie externe qui en a besoin est arrivée dans le bucket le {when} ; nécessaire jusqu'au {until}", + "backupsOffhostLineUnknown": "Clé {key} : la dernière copie externe qui en a besoin est arrivée dans le bucket le {when}. La durée de conservation du bucket est inconnue : gardez la clé jusqu'à être sûr que cette copie a disparu.", + "backupsNone": "Aucune sauvegarde n'a encore enregistré les clés dont elle a besoin.", + "backupsStillNeed": "Toutes les lignes sont renouvelées, mais des sauvegardes ont encore besoin de {keys}", + "backupsUnrecordedBadge": "Toutes les lignes sont renouvelées, mais d'anciennes sauvegardes peuvent encore avoir besoin d'une clé retirée", + "backupsDetail": "La rotation rechiffre chaque ligne, mais une sauvegarde garde la clé sous laquelle son contenu a été écrit, et sa restauration en a besoin. Gardez une clé retirée dans ENCRYPTION_KEYS tant qu'elle figure ci-dessous." } }, "carrier": "Opérateur", diff --git a/messages/it.json b/messages/it.json index 39e7a2901..34729a882 100644 --- a/messages/it.json +++ b/messages/it.json @@ -7276,7 +7276,12 @@ "neverDetail": "Per questo account non è arrivato nulla. Ultimo controllo {when}.", "unknownDetail": "Nessun giro notturno ha ancora registrato questo account.", "age": "{hours} h fa", - "scheduleNote": "Il caricamento gira ogni {hours} h. Recente significa entro un giro, attesa che un giro non ha prodotto nulla, obsoleta che non lo hanno fatto due." + "scheduleNote": "Il caricamento gira ogni {hours} h. Recente significa entro un giro, attesa che un giro non ha prodotto nulla, obsoleta che non lo hanno fatto due.", + "lifecycleConfigured": "Il bucket elimina le copie dopo {days} giorni con la propria regola di ciclo di vita.", + "lifecycleMissing": "Il bucket non ha una regola di ciclo di vita, quindi ogni copia notturna viene conservata per sempre. Aggiungi una regola che faccia scadere gli oggetti dopo BACKUP_RETENTION_DAYS.", + "lifecycleUnknown": "Non è stato possibile leggere se il bucket elimina le copie vecchie: la credenziale non può leggere la sua regola di ciclo di vita. Verifica che una regola faccia scadere gli oggetti dopo BACKUP_RETENTION_DAYS.", + "pendingDeletions": "Copie di {count} account eliminati o svuotati sono ancora nel bucket e vengono rimosse.", + "pendingDeletionsFailing": "Copie di {count} account eliminati o svuotati sono ancora nel bucket perché ha rifiutato di eliminarle ({error}). Concedi DeleteObject alla credenziale; l'eliminazione viene ritentata ogni notte." }, "restoreStarted": "Ripristino avviato. L'avanzamento compare sopra l'elenco.", "restoreActive": "Per questo account è già in corso un ripristino. Aspetta che finisca.", @@ -7364,7 +7369,17 @@ "completed": "Completato", "failed": "Fallito" }, - "coverageDetail": "Vengono mostrati solo gli id di chiave e i conteggi, mai il materiale della chiave." + "coverageDetail": "Vengono mostrati solo gli id di chiave e i conteggi, mai il materiale della chiave.", + "backupsTitle": "Chiavi di cui i backup hanno ancora bisogno", + "backupsDescription": "Una chiave ritirata resta necessaria finché un backup qui sotto la elenca ancora.", + "backupsStoredLine": "Chiave {key}: {count} copie salvate, la più vecchia del {when}", + "backupsUnrecordedLine": "{count} copie salvate sono state scritte prima che i backup registrassero le loro chiavi (la più vecchia il {when}). Possono richiedere qualsiasi chiave esistente allora: conserva tutte le vecchie chiavi finché non vengono sostituite o eliminate.", + "backupsOffhostLine": "Chiave {key}: l'ultima copia esterna che la richiede è entrata nel bucket il {when}; necessaria fino al {until}", + "backupsOffhostLineUnknown": "Chiave {key}: l'ultima copia esterna che la richiede è entrata nel bucket il {when}. La scadenza del bucket è sconosciuta, quindi conserva la chiave finché non sai che quella copia non c'è più.", + "backupsNone": "Nessun backup ha ancora registrato le chiavi di cui ha bisogno.", + "backupsStillNeed": "Tutte le righe sono ruotate, ma i backup hanno ancora bisogno di {keys}", + "backupsUnrecordedBadge": "Tutte le righe sono ruotate, ma backup più vecchi potrebbero ancora richiedere una chiave ritirata", + "backupsDetail": "La rotazione cifra di nuovo ogni riga, ma un backup conserva la chiave con cui è stato scritto il suo contenuto, e ripristinarlo richiede quella chiave. Tieni una chiave ritirata in ENCRYPTION_KEYS finché compare qui sotto." } }, "carrier": "Operatore", diff --git a/messages/ko.json b/messages/ko.json index 38a032204..eea66600e 100644 --- a/messages/ko.json +++ b/messages/ko.json @@ -7255,7 +7255,12 @@ "neverDetail": "이 계정으로 도착한 것이 없습니다. 마지막 확인 {when}.", "unknownDetail": "아직 어떤 야간 실행도 이 계정을 기록하지 않았습니다.", "age": "{hours}시간 전", - "scheduleNote": "업로드는 {hours}시간마다 실행됩니다. 최신은 한 번의 실행 이내, 지연은 한 번이 아무것도 만들지 못한 상태, 오래됨은 두 번이 그런 상태입니다." + "scheduleNote": "업로드는 {hours}시간마다 실행됩니다. 최신은 한 번의 실행 이내, 지연은 한 번이 아무것도 만들지 못한 상태, 오래됨은 두 번이 그런 상태입니다.", + "lifecycleConfigured": "버킷의 수명 주기 규칙에 따라 복사본이 {days}일 후 삭제됩니다.", + "lifecycleMissing": "버킷에 수명 주기 규칙이 없어 매일 밤의 복사본이 영구히 보관됩니다. BACKUP_RETENTION_DAYS 후 객체를 만료시키는 규칙을 추가하세요.", + "lifecycleUnknown": "버킷이 오래된 복사본을 삭제하는지 읽을 수 없었습니다. 자격 증명이 수명 주기 규칙을 읽지 못할 수 있습니다. BACKUP_RETENTION_DAYS 후 객체를 만료시키는 규칙이 있는지 확인하세요.", + "pendingDeletions": "삭제되었거나 비워진 계정 {count}개의 복사본이 아직 버킷에 있으며 제거 중입니다.", + "pendingDeletionsFailing": "버킷이 삭제를 거부하여 삭제되었거나 비워진 계정 {count}개의 복사본이 아직 남아 있습니다 ({error}). 자격 증명에 DeleteObject 권한을 부여하세요. 삭제는 매일 밤 다시 시도됩니다." }, "restoreStarted": "복원을 시작했어요. 진행 상황은 목록 위에 표시돼요.", "restoreActive": "이 계정의 복원이 이미 진행 중이에요. 끝날 때까지 기다려 주세요.", @@ -7343,7 +7348,17 @@ "completed": "완료", "failed": "실패" }, - "coverageDetail": "키 id와 개수만 보여줘요 — 키 자체는 절대 표시하지 않아요." + "coverageDetail": "키 id와 개수만 보여줘요 — 키 자체는 절대 표시하지 않아요.", + "backupsTitle": "백업에 아직 필요한 키", + "backupsDescription": "아래 백업 중 하나라도 표시하는 동안 폐기한 키는 계속 필요합니다.", + "backupsStoredLine": "키 {key}: 저장된 복사본 {count}개, 가장 오래된 것은 {when}", + "backupsUnrecordedLine": "저장된 복사본 {count}개는 백업이 키를 기록하기 전에 작성되었습니다(가장 오래된 것은 {when}). 당시 존재하던 어떤 키든 필요할 수 있으니, 교체되거나 삭제될 때까지 이전 키를 모두 유지하세요.", + "backupsOffhostLine": "키 {key}: 이 키가 필요한 마지막 외부 복사본이 {when}에 버킷에 저장되었습니다. {until}까지 필요합니다", + "backupsOffhostLineUnknown": "키 {key}: 이 키가 필요한 마지막 외부 복사본이 {when}에 버킷에 저장되었습니다. 버킷의 만료 기간을 알 수 없으므로 그 복사본이 사라진 것을 확인할 때까지 키를 유지하세요.", + "backupsNone": "아직 필요한 키를 기록한 백업이 없습니다.", + "backupsStillNeed": "모든 행이 회전되었지만 백업에 아직 {keys}가 필요합니다", + "backupsUnrecordedBadge": "모든 행이 회전되었지만 이전 백업에는 아직 폐기한 키가 필요할 수 있습니다", + "backupsDetail": "회전은 모든 행을 다시 암호화하지만, 백업은 내용이 기록된 키를 그대로 유지하며 복원에는 그 키가 필요합니다. 아래에 더 이상 표시되지 않을 때까지 폐기한 키를 ENCRYPTION_KEYS에 유지하세요." } }, "carrier": "통신사", diff --git a/messages/pl.json b/messages/pl.json index ec48365ea..991abd5de 100644 --- a/messages/pl.json +++ b/messages/pl.json @@ -7276,7 +7276,12 @@ "neverDetail": "Dla tego konta nic nie dotarło. Ostatnie sprawdzenie {when}.", "unknownDetail": "Żaden nocny przebieg nie zarejestrował jeszcze tego konta.", "age": "{hours} h temu", - "scheduleNote": "Wysyłka działa co {hours} h. Świeża znaczy w obrębie jednego przebiegu, zaległa że jeden nic nie dał, przeterminowana że dwa." + "scheduleNote": "Wysyłka działa co {hours} h. Świeża znaczy w obrębie jednego przebiegu, zaległa że jeden nic nie dał, przeterminowana że dwa.", + "lifecycleConfigured": "Bucket usuwa kopie po {days} dniach zgodnie z własną regułą cyklu życia.", + "lifecycleMissing": "Bucket nie ma reguły cyklu życia, więc każda nocna kopia zostaje na zawsze. Dodaj regułę, która usuwa obiekty po BACKUP_RETENTION_DAYS.", + "lifecycleUnknown": "Nie udało się odczytać, czy bucket usuwa stare kopie: poświadczenia nie mogą odczytać jego reguły cyklu życia. Sprawdź, czy reguła usuwa obiekty po BACKUP_RETENTION_DAYS.", + "pendingDeletions": "Kopie {count} usuniętych lub wyczyszczonych kont są jeszcze w buckecie i są usuwane.", + "pendingDeletionsFailing": "Kopie {count} usuniętych lub wyczyszczonych kont są jeszcze w buckecie, bo odmówił ich usunięcia ({error}). Nadaj poświadczeniom uprawnienie DeleteObject; usuwanie jest ponawiane co noc." }, "restoreStarted": "Przywracanie rozpoczęte. Postęp widać nad listą.", "restoreActive": "Dla tego konta trwa już przywracanie. Poczekaj, aż się skończy.", @@ -7364,7 +7369,17 @@ "completed": "Zakończono", "failed": "Niepowodzenie" }, - "coverageDetail": "Pokazywane są tylko identyfikatory kluczy i liczby — nigdy materiał klucza." + "coverageDetail": "Pokazywane są tylko identyfikatory kluczy i liczby — nigdy materiał klucza.", + "backupsTitle": "Klucze, których kopie nadal potrzebują", + "backupsDescription": "Wycofany klucz jest potrzebny, dopóki którakolwiek kopia poniżej go wymienia.", + "backupsStoredLine": "Klucz {key}: {count} zapisanych kopii, najstarsza z {when}", + "backupsUnrecordedLine": "{count} zapisanych kopii powstało, zanim kopie zaczęły zapisywać swoje klucze (najstarsza {when}). Mogą potrzebować każdego klucza, który wtedy istniał: zachowaj wszystkie stare klucze, dopóki nie zostaną zastąpione lub usunięte.", + "backupsOffhostLine": "Klucz {key}: ostatnia kopia zewnętrzna, która go potrzebuje, trafiła do bucketu {when}; potrzebny tam do {until}", + "backupsOffhostLineUnknown": "Klucz {key}: ostatnia kopia zewnętrzna, która go potrzebuje, trafiła do bucketu {when}. Nie wiadomo, jak długo bucket przechowuje kopie, więc zachowaj klucz, dopóki nie masz pewności, że tej kopii już nie ma.", + "backupsNone": "Żadna kopia nie zapisała jeszcze kluczy, których potrzebuje.", + "backupsStillNeed": "Wszystkie wiersze są zrotowane, ale kopie nadal potrzebują {keys}", + "backupsUnrecordedBadge": "Wszystkie wiersze są zrotowane, ale starsze kopie mogą nadal potrzebować wycofanego klucza", + "backupsDetail": "Rotacja ponownie szyfruje każdy wiersz, ale kopia zachowuje klucz, którym zapisano jej zawartość, a jej przywrócenie wymaga tego klucza. Trzymaj wycofany klucz w ENCRYPTION_KEYS, dopóki widnieje poniżej." } }, "carrier": "Operator", diff --git a/prisma/migrations/0358_backup_key_ids_and_offhost_purge/migration.sql b/prisma/migrations/0358_backup_key_ids_and_offhost_purge/migration.sql new file mode 100644 index 000000000..09a577e69 --- /dev/null +++ b/prisma/migrations/0358_backup_key_ids_and_offhost_purge/migration.sql @@ -0,0 +1,37 @@ +-- Backups: which keys their content needs, and off-host copies that leave +-- with the account. +-- +-- 1. A stored backup carries encrypted columns as the stored ciphertext, and +-- key rotation cannot reach inside a copy. Each copy now records the key +-- ids its content was written under, so the admin encryption view can say +-- which retired key is still needed and by how old a copy. Copies written +-- before this release are marked unrecorded rather than assumed clean. +ALTER TABLE "data_backups" + ADD COLUMN "inner_key_ids" TEXT[] NOT NULL DEFAULT ARRAY[]::TEXT[], + ADD COLUMN "inner_key_ids_recorded" BOOLEAN NOT NULL DEFAULT false; + +-- 2. The same fact for the off-host bucket, per key rather than per object: +-- when the nightly run last wrote an object that needed each key. +CREATE TABLE "offhost_backup_key_use" ( + "key_id" TEXT NOT NULL, + "first_written_at" TIMESTAMP(3) NOT NULL, + "last_written_at" TIMESTAMP(3) NOT NULL, + CONSTRAINT "offhost_backup_key_use_pkey" PRIMARY KEY ("key_id") +); + +-- 3. Accounts whose off-host objects have to be deleted: written with the +-- account deletion or the data wipe, removed once the bucket holds nothing +-- for the account. No foreign key, because a deleted account's row is gone +-- by the time the job runs. +CREATE TABLE "offhost_purge_requests" ( + "id" TEXT NOT NULL, + "subject_id" TEXT NOT NULL, + "reason" TEXT NOT NULL, + "requested_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "attempts" INTEGER NOT NULL DEFAULT 0, + "last_attempt_at" TIMESTAMP(3), + "last_failure" TEXT, + CONSTRAINT "offhost_purge_requests_pkey" PRIMARY KEY ("id") +); +CREATE INDEX "offhost_purge_requests_requested_at_idx" + ON "offhost_purge_requests"("requested_at"); diff --git a/prisma/schema.prisma b/prisma/schema.prisma index 7d725e8d3..1bb86bb34 100644 --- a/prisma/schema.prisma +++ b/prisma/schema.prisma @@ -4970,19 +4970,29 @@ model PushSubscription { // ─── Data Backups ──────────────────────────────────────────── model DataBackup { - id String @id @default(cuid()) - userId String @map("user_id") - type String @default("WEEKLY_AUTO") // WEEKLY_AUTO, MANUAL_UPLOAD_ + id String @id @default(cuid()) + userId String @map("user_id") + type String @default("WEEKLY_AUTO") // WEEKLY_AUTO, MANUAL_UPLOAD_ /// The whole encrypted copy as one value: how every backup was stored up /// to v1.39.1. NULL for a copy stored in pieces (`chunks`), which is how /// every backup is written from v1.39.2 on. A row carries one or the other. - data String? + data String? /// How many pieces a chunked copy has, and the random id sealed into every /// one of them (`src/lib/export/backup-chunks.ts`). The reader checks both /// against the pieces before it releases a byte; NULL on a single-value copy. - chunkCount Int? @map("chunk_count") - chunkStreamId String? @map("chunk_stream_id") - createdAt DateTime @default(now()) @map("created_at") + chunkCount Int? @map("chunk_count") + chunkStreamId String? @map("chunk_stream_id") + /// The application key ids the ciphertext INSIDE this copy was written + /// under, recorded as the copy is written (`createBackupKeyIdTextScanner`). + /// Key rotation re-seals the copy's envelope but not what is inside it, so + /// a key stays needed while any copy lists it. NULL for a copy written + /// before v1.39.3, which the admin encryption view counts as unrecorded. + /// Consumer: `GET /api/admin/encryption/status` (`backups.stored`). + innerKeyIds String[] @default([]) @map("inner_key_ids") + /// Whether `innerKeyIds` was recorded; false on every copy written before + /// v1.39.3. Consumer: the same status read (`backups.unrecorded`). + innerKeyIdsRecorded Boolean @default(false) @map("inner_key_ids_recorded") + createdAt DateTime @default(now()) @map("created_at") user User @relation(fields: [userId], references: [id], onDelete: Cascade) chunks DataBackupChunk[] @@ -5148,6 +5158,43 @@ model OffhostBackupState { @@map("offhost_backup_state") } +/// Per application key id, when the nightly off-host run last put an object +/// in the bucket whose inner ciphertext needed that key. The bucket's +/// lifecycle rule keeps each object for its retention, so a retired key is +/// needed until that long after this date. Not per account: the question it +/// answers is about keys. Written by `runOffhostBackup`; consumer: +/// `GET /api/admin/encryption/status` (`backups.offhost`). +model OffhostBackupKeyUse { + keyId String @id @map("key_id") + firstWrittenAt DateTime @map("first_written_at") + lastWrittenAt DateTime @map("last_written_at") + + @@map("offhost_backup_key_use") +} + +/// An account whose off-host copies have to leave the bucket: it was deleted, +/// or its data was wiped. Written in the same transaction as the deletion, so +/// the request cannot be lost between the two, and deleted once every object +/// for the account is gone. `subjectId` is deliberately not a relation: for a +/// deleted account the user row is already gone. Written by +/// `requestOffhostPurge`; consumer: `processOffhostPurges` (the +/// `offhost-backup-purge` job) and the off-host card's pending count behind +/// `GET /api/admin/backups`. +model OffhostPurgeRequest { + id String @id @default(cuid()) + subjectId String @map("subject_id") + /// `account_deleted`, `data_wiped`, `managed_profile_deleted` or `admin_wipe`. + reason String + requestedAt DateTime @default(now()) @map("requested_at") + attempts Int @default(0) + lastAttemptAt DateTime? @map("last_attempt_at") + /// The bucket's last refusal, trimmed; shown to the operator. + lastFailure String? @map("last_failure") + + @@index([requestedAt]) + @@map("offhost_purge_requests") +} + // ─── Apple Health export.zip import (v1.4.34) ─────────────── // // Mirrors a `pg-boss` `apple-health-import` job's lifecycle so the diff --git a/src/app/api/admin/backups/__tests__/route.test.ts b/src/app/api/admin/backups/__tests__/route.test.ts index 70b583451..350c4756b 100644 --- a/src/app/api/admin/backups/__tests__/route.test.ts +++ b/src/app/api/admin/backups/__tests__/route.test.ts @@ -29,12 +29,23 @@ vi.mock("@/lib/db", () => ({ prisma: { $queryRaw: (...a: unknown[]) => queryRawMock(...a), user: { findMany: (...a: unknown[]) => findManyMock(...a) }, + offhostPurgeRequest: { + count: async () => 2, + findFirst: async () => ({ + requestedAt: new Date("2026-09-20T10:00:00.000Z"), + lastFailure: "AccessDenied", + }), + }, }, })); const configuredMock = vi.fn(() => true); vi.mock("@/lib/jobs/offhost-backup", () => ({ offhostBackupConfigured: () => configuredMock(), + probeOffhostLifecycle: async () => ({ + state: "missing", + expirationDays: null, + }), })); import { GET } from "../route"; @@ -139,10 +150,12 @@ describe("GET /api/admin/backups — off-host freshness", () => { const body = await read(); - expect(body.offhost).toEqual({ + expect(body.offhost).toMatchObject({ configured: false, periodHours: 24, rows: [], + // No bucket to ask on a host that is not configured. + lifecycle: { state: "unknown", expirationDays: null }, }); }); @@ -196,4 +209,20 @@ describe("GET /api/admin/backups — off-host freshness", () => { ]); expect(JSON.stringify(body.offhost)).not.toMatch(/bucket|endpoint|secret/i); }); + + it("carries the bucket's lifecycle verdict and the deletions it still owes", async () => { + configuredMock.mockReturnValue(true); + queryRawMock.mockResolvedValue([]); + findManyMock.mockResolvedValue([]); + const body = await read(); + expect(body.offhost.lifecycle).toEqual({ + state: "missing", + expirationDays: null, + }); + expect(body.offhost.pendingDeletions).toEqual({ + count: 2, + oldestRequestedAt: "2026-09-20T10:00:00.000Z", + lastFailure: "AccessDenied", + }); + }); }); diff --git a/src/app/api/admin/backups/route.ts b/src/app/api/admin/backups/route.ts index 09d9cc9f6..2a1939e08 100644 --- a/src/app/api/admin/backups/route.ts +++ b/src/app/api/admin/backups/route.ts @@ -27,7 +27,10 @@ import { annotate } from "@/lib/logging/context"; import { DATA_BACKUP_QUEUE } from "@/lib/jobs/data-backup-policy"; import { readLastQueueRun } from "@/lib/jobs/job-failures"; import { summariseBackupSchedule } from "@/lib/jobs/backup-schedule-status"; -import { offhostBackupConfigured } from "@/lib/jobs/offhost-backup"; +import { + offhostBackupConfigured, + probeOffhostLifecycle, +} from "@/lib/jobs/offhost-backup"; import { classifyOffhostBackup, OFFHOST_BACKUP_PERIOD_HOURS, @@ -164,6 +167,17 @@ export const GET = apiHandler(async () => { a.username.localeCompare(b.username), ); + const [purgeCount, oldestPurge, lifecycle] = await Promise.all([ + prisma.offhostPurgeRequest.count(), + prisma.offhostPurgeRequest.findFirst({ + orderBy: { requestedAt: "asc" }, + select: { requestedAt: true, lastFailure: true }, + }), + configured + ? probeOffhostLifecycle() + : Promise.resolve({ state: "unknown" as const, expirationDays: null }), + ]); + const payload: BackupsList = { rows: list, // Matches the retention window the backup-prune job enforces so the @@ -182,6 +196,12 @@ export const GET = apiHandler(async () => { // An unconfigured host has nothing to list, and the card says so rather // than painting an empty table that reads as "no problems". rows: configured ? offhostRows : [], + pendingDeletions: { + count: purgeCount, + oldestRequestedAt: oldestPurge?.requestedAt.toISOString() ?? null, + lastFailure: oldestPurge?.lastFailure ?? null, + }, + lifecycle, }, }; diff --git a/src/app/api/admin/data/route.ts b/src/app/api/admin/data/route.ts index 9fcbb9afe..77bddb7e9 100644 --- a/src/app/api/admin/data/route.ts +++ b/src/app/api/admin/data/route.ts @@ -1,3 +1,8 @@ +import { + kickOffhostPurge, + PURGE_ALL_SUBJECTS, + requestOffhostPurge, +} from "@/lib/jobs/offhost-purge"; import { prisma } from "@/lib/db"; import { apiHandler, requireAdmin } from "@/lib/api-handler"; import { auditLog } from "@/lib/auth/audit"; @@ -108,6 +113,9 @@ export const DELETE = apiHandler(async (request: NextRequest) => { // columns does not. Same classification contract as the model list. await tx.user.updateMany({ data: USER_RESET }); + // Every account's nightly off-host copies go too. + await requestOffhostPurge(tx, PURGE_ALL_SUBJECTS, "admin_wipe"); + return perModel; }, { @@ -138,6 +146,7 @@ export const DELETE = apiHandler(async (request: NextRequest) => { // v1.16.9 — the wipe touched every user's rows; clear every cache // bucket so no per-user payload survives the reset. invalidateAllCaches(); + await kickOffhostPurge(); return apiSuccess({ cleared: true, deletedRows, models: counts }); }); diff --git a/src/app/api/admin/encryption/status/route.ts b/src/app/api/admin/encryption/status/route.ts index 76cc2b02f..4bda28af8 100644 --- a/src/app/api/admin/encryption/status/route.ts +++ b/src/app/api/admin/encryption/status/route.ts @@ -19,6 +19,10 @@ import { apiHandler, requireAdmin } from "@/lib/api-handler"; import { apiSuccess } from "@/lib/api-response"; import { annotate } from "@/lib/logging/context"; import { getConfiguredKeyIds } from "@/lib/crypto"; +import { + offhostBackupConfigured, + probeOffhostLifecycle, +} from "@/lib/jobs/offhost-backup"; import { scanCorpus, type CorpusClient, @@ -100,6 +104,100 @@ async function readRotationState(): Promise { }; } +/** + * Which keys the backups still need. + * + * Rotation re-encrypts every row in the database and re-seals every stored + * backup's envelope, but the content INSIDE a backup keeps the key it was + * written under: a restore writes it back verbatim. So "every row is on the + * active key" is not the same as "the old key can go". This answers the second + * question per key id, from what each copy recorded as it was written: the + * stored copies that need it and the oldest of them, and for the off-host + * bucket the last night an object needing it went in, plus how long the + * bucket's lifecycle rule keeps objects after that. + */ +export interface BackupKeyNeeds { + stored: Array<{ keyId: string; copies: number; oldestAt: string }>; + /** Copies written before v1.39.3, which recorded no key ids. */ + unrecorded: { copies: number; oldestAt: string | null }; + offhost: Array<{ + keyId: string; + firstWrittenAt: string; + lastWrittenAt: string; + /** When the last object needing this key expires; null when unknown. */ + neededUntil: string | null; + }>; + /** The bucket's expiry, when it has one the credential may read. */ + offhostExpirationDays: number | null; + /** Retired key ids some backup still needs (never the active one). */ + retiredKeysStillNeeded: string[]; +} + +async function readBackupKeyNeeds( + activeKeyId: string, + now: Date, +): Promise { + const [stored, unrecorded, offhostRows] = await Promise.all([ + prisma.$queryRaw< + Array<{ key_id: string; copies: number; oldest_at: Date }> + >` + SELECT k AS key_id, count(*)::int AS copies, min(created_at) AS oldest_at + FROM data_backups, unnest(inner_key_ids) AS k + WHERE inner_key_ids_recorded + GROUP BY k + ORDER BY k + `, + prisma.dataBackup.aggregate({ + where: { innerKeyIdsRecorded: false }, + _count: { _all: true }, + _min: { createdAt: true }, + }), + prisma.offhostBackupKeyUse.findMany({ orderBy: { keyId: "asc" } }), + ]); + const lifecycle = + offhostRows.length > 0 && offhostBackupConfigured() + ? await probeOffhostLifecycle() + : null; + const expirationDays = + lifecycle?.state === "configured" ? lifecycle.expirationDays : null; + + const offhost = offhostRows.map((row) => ({ + keyId: row.keyId, + firstWrittenAt: row.firstWrittenAt.toISOString(), + lastWrittenAt: row.lastWrittenAt.toISOString(), + neededUntil: + expirationDays === null + ? null + : new Date( + row.lastWrittenAt.getTime() + (expirationDays + 1) * 86_400_000, + ).toISOString(), + })); + + const needed = new Set(); + for (const row of stored) needed.add(row.key_id); + for (const row of offhost) { + if (row.neededUntil === null || Date.parse(row.neededUntil) > +now) { + needed.add(row.keyId); + } + } + needed.delete(activeKeyId); + + return { + stored: stored.map((row) => ({ + keyId: row.key_id, + copies: row.copies, + oldestAt: row.oldest_at.toISOString(), + })), + unrecorded: { + copies: unrecorded._count._all, + oldestAt: unrecorded._min.createdAt?.toISOString() ?? null, + }, + offhost, + offhostExpirationDays: expirationDays, + retiredKeysStillNeeded: [...needed].sort(), + }; +} + export const GET = apiHandler(async () => { await requireAdmin(); annotate({ action: { name: "admin.encryption.status" } }); @@ -114,6 +212,7 @@ export const GET = apiHandler(async () => { } const rotation = await readRotationState(); + const backups = await readBackupKeyNeeds(scan.activeKeyId, new Date()); annotate({ meta: { @@ -133,5 +232,14 @@ export const GET = apiHandler(async () => { staleRows: scan.staleRows, columns: scan.columns, rotation, + backups, + // Every row rotated AND no backup still needs a retired key: the one + // state in which dropping an old key loses nothing. Copies that recorded + // no key ids count against it, because they may hold any key that + // existed when they were written. + safeToDropRetiredKeys: + scan.rotationComplete && + backups.retiredKeysStillNeeded.length === 0 && + backups.unrecorded.copies === 0, }); }); diff --git a/src/app/api/settings/account/route.ts b/src/app/api/settings/account/route.ts index 4f863defc..60e06374a 100644 --- a/src/app/api/settings/account/route.ts +++ b/src/app/api/settings/account/route.ts @@ -8,6 +8,10 @@ import { MFA_STEP_UP_MAX_AGE_SECONDS, } from "@/lib/api-handler"; import { annotate } from "@/lib/logging/context"; +import { + kickOffhostPurge, + requestOffhostPurge, +} from "@/lib/jobs/offhost-purge"; import { deleteGuardianAccountWithLifecycle, LastManagedGuardianError, @@ -101,6 +105,10 @@ export const DELETE = apiHandler(async (request: NextRequest) => { // genuinely complete erasure. await tx.auditLog.deleteMany({ where: { userId } }); + // The nightly off-host copies go with the account. The request commits + // with the deletion; the purge job deletes the objects. + await requestOffhostPurge(tx, userId, "account_deleted"); + // Delete user — all other related data is removed via onDelete: Cascade. await tx.user.delete({ where: { id: userId } }); }); @@ -115,6 +123,8 @@ export const DELETE = apiHandler(async (request: NextRequest) => { throw error; } + await kickOffhostPurge(); + annotate({ action: { name: "settings.account.delete" }, meta: { username }, diff --git a/src/app/api/settings/data/route.ts b/src/app/api/settings/data/route.ts index 409320ccc..2c3187e0f 100644 --- a/src/app/api/settings/data/route.ts +++ b/src/app/api/settings/data/route.ts @@ -1,5 +1,9 @@ import { prisma } from "@/lib/db"; import { auditLog } from "@/lib/auth/audit"; +import { + kickOffhostPurge, + requestOffhostPurge, +} from "@/lib/jobs/offhost-purge"; import { apiSuccess, apiError, getClientIp } from "@/lib/api-response"; import { NextRequest } from "next/server"; import { @@ -101,6 +105,11 @@ export const DELETE = apiHandler(async (request: NextRequest) => { // does not. Same classification contract as the model list. await tx.user.update({ where: { id: userId }, data: USER_RESET }); + // The nightly off-host copies of the record are part of "all your + // data". The request commits with the wipe; the purge job deletes + // the objects and removes it. + await requestOffhostPurge(tx, userId, "data_wiped"); + return perModel; }, { @@ -144,6 +153,7 @@ export const DELETE = apiHandler(async (request: NextRequest) => { // v1.16.9 — every cached payload was built on the rows just deleted; // hard-evict the user's buckets so no surface serves pre-wipe data. invalidateUserData(userId); + await kickOffhostPurge(); return apiSuccess({ cleared: true, deletedRows, models: counts }); }); diff --git a/src/components/admin/__tests__/offhost-backups-section.test.tsx b/src/components/admin/__tests__/offhost-backups-section.test.tsx index d3dd46985..1bc8a886d 100644 --- a/src/components/admin/__tests__/offhost-backups-section.test.tsx +++ b/src/components/admin/__tests__/offhost-backups-section.test.tsx @@ -23,6 +23,11 @@ vi.mock("@tanstack/react-query", () => ({ import { I18nProvider } from "@/lib/i18n/context"; import { OffhostBackupsSection } from "../offhost-backups-section"; +const QUIET = { + pendingDeletions: { count: 0, oldestRequestedAt: null, lastFailure: null }, + lifecycle: { state: "configured" as const, expirationDays: 30 }, +}; + function render(data: BackupsList["offhost"] | null): string { queryResult.current = { data: @@ -46,7 +51,12 @@ function render(data: BackupsList["offhost"] | null): string { describe("", () => { it("says off-host backup is not configured rather than showing an empty list", () => { - const html = render({ configured: false, periodHours: 24, rows: [] }); + const html = render({ + configured: false, + periodHours: 24, + rows: [], + ...QUIET, + }); expect(html).toContain("Off-host backup is not configured"); expect(html).not.toContain('data-slot="offhost-backup-rows"'); }); @@ -55,6 +65,7 @@ describe("", () => { const html = render({ configured: true, periodHours: 24, + ...QUIET, rows: [ { userId: "u1", @@ -109,4 +120,33 @@ describe("", () => { // Never the bucket's own coordinates. expect(html).not.toMatch(/secret|access[- ]key/i); }); + + it("says when the bucket keeps copies forever, and when deletions are stuck", () => { + const html = render({ + configured: true, + periodHours: 24, + rows: [], + pendingDeletions: { + count: 2, + oldestRequestedAt: "2026-09-20T10:00:00.000Z", + lastFailure: "AccessDenied", + }, + lifecycle: { state: "missing", expirationDays: null }, + }); + expect(html).toContain('data-lifecycle-state="missing"'); + expect(html).toContain("no lifecycle rule"); + expect(html).toContain('data-slot="offhost-pending-deletions"'); + expect(html).toContain("AccessDenied"); + }); + + it("states the expiry the bucket enforces", () => { + const html = render({ + configured: true, + periodHours: 24, + rows: [], + ...QUIET, + }); + expect(html).toContain("after 30 days"); + expect(html).not.toContain('data-slot="offhost-pending-deletions"'); + }); }); diff --git a/src/components/admin/encryption-section.tsx b/src/components/admin/encryption-section.tsx index 7b043fd78..dda5efb26 100644 --- a/src/components/admin/encryption-section.tsx +++ b/src/components/admin/encryption-section.tsx @@ -13,7 +13,13 @@ */ import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query"; -import { KeyRound, Loader2, RotateCw, ShieldCheck } from "lucide-react"; +import { + Archive, + KeyRound, + Loader2, + RotateCw, + ShieldCheck, +} from "lucide-react"; import { toast } from "sonner"; import { AlertDialog, @@ -32,7 +38,7 @@ import { StatTile } from "@/components/admin/_stat-tile"; import { Button } from "@/components/ui/button"; import { SettingsCard } from "@/components/settings/settings-card"; import { SettingsCardHeader } from "@/components/settings/_card-header"; -import { useTranslations } from "@/lib/i18n/context"; +import { useFormatters, useTranslations } from "@/lib/i18n/context"; import { queryKeys } from "@/lib/query-keys"; import { apiFetchRaw, apiGet } from "@/lib/api/api-fetch"; import { getApiErrorMessage } from "./_shared"; @@ -60,10 +66,25 @@ interface EncryptionStatus { lastCompletedAt: string | null; lastResult: { scanned: number; rotated: number; errors: number } | null; }; + /** Which keys the backups still need; see the status route. */ + backups: { + stored: Array<{ keyId: string; copies: number; oldestAt: string }>; + unrecorded: { copies: number; oldestAt: string | null }; + offhost: Array<{ + keyId: string; + firstWrittenAt: string; + lastWrittenAt: string; + neededUntil: string | null; + }>; + offhostExpirationDays: number | null; + retiredKeysStillNeeded: string[]; + }; + safeToDropRetiredKeys: boolean; } export function EncryptionSection() { const { t } = useTranslations(); + const fmt = useFormatters(); const queryClient = useQueryClient(); const statusQuery = useQuery({ @@ -166,10 +187,20 @@ export function EncryptionSection() { value={s.staleRows.toLocaleString()} /> - {s.rotationComplete ? ( + {s.safeToDropRetiredKeys ? ( {t("admin.section.encryption.safeToDropLegacy")} + ) : s.rotationComplete ? ( + // Every row is on the active key, and that is not the whole + // answer: a backup keeps the key its content was written under. + + {s.backups.retiredKeysStillNeeded.length > 0 + ? t("admin.section.encryption.backupsStillNeed", { + keys: s.backups.retiredKeysStillNeeded.join(", "), + }) + : t("admin.section.encryption.backupsUnrecordedBadge")} + ) : ( {t("admin.section.encryption.rotationIncomplete", { @@ -179,6 +210,56 @@ export function EncryptionSection() { )} + {/* ── Keys the backups still need ───────────────────────────── */} + + +

{t("admin.section.encryption.backupsDetail")}

+
    + {s.backups.stored.map((row) => ( +
  • + {t("admin.section.encryption.backupsStoredLine", { + key: row.keyId, + count: row.copies, + when: fmt.dateTime(row.oldestAt), + })} +
    • + ))} + {s.backups.unrecorded.copies > 0 && s.backups.unrecorded.oldestAt ? ( +
  • + {t("admin.section.encryption.backupsUnrecordedLine", { + count: s.backups.unrecorded.copies, + when: fmt.dateTime(s.backups.unrecorded.oldestAt), + })} +
    • + ) : null} + {s.backups.offhost.map((row) => ( +
  • + {row.neededUntil + ? t("admin.section.encryption.backupsOffhostLine", { + key: row.keyId, + when: fmt.dateTime(row.lastWrittenAt), + until: fmt.dateTime(row.neededUntil), + }) + : t("admin.section.encryption.backupsOffhostLineUnknown", { + key: row.keyId, + when: fmt.dateTime(row.lastWrittenAt), + })} +
    • + ))} + {s.backups.stored.length === 0 && + s.backups.unrecorded.copies === 0 && + s.backups.offhost.length === 0 ? ( +
  • + {t("admin.section.encryption.backupsNone")} +
    • + ) : null} +
+ + {/* ── Rotation status + trigger ──────────────────────────────── */} +

+ {offhost.lifecycle.state === "configured" + ? t("admin.section.backups.offhost.lifecycleConfigured", { + days: offhost.lifecycle.expirationDays ?? 0, + }) + : offhost.lifecycle.state === "missing" + ? t("admin.section.backups.offhost.lifecycleMissing") + : t("admin.section.backups.offhost.lifecycleUnknown")} +

+ {offhost.pendingDeletions.count > 0 ? ( +

+ {offhost.pendingDeletions.lastFailure + ? t("admin.section.backups.offhost.pendingDeletionsFailing", { + count: offhost.pendingDeletions.count, + error: offhost.pendingDeletions.lastFailure, + }) + : t("admin.section.backups.offhost.pendingDeletions", { + count: offhost.pendingDeletions.count, + })} +

+ ) : null} ) : null} diff --git a/src/lib/data-wipe/wipe-plan.ts b/src/lib/data-wipe/wipe-plan.ts index 3bb1bcf60..a845f6243 100644 --- a/src/lib/data-wipe/wipe-plan.ts +++ b/src/lib/data-wipe/wipe-plan.ts @@ -293,6 +293,10 @@ export const INSTANCE_SCOPED: Readonly> = { AppSettings: "instance-wide configuration singleton", RateLimit: "instance-wide rate-limit buckets, keyed by string, self-expiring", HostMetric: "instance-wide host telemetry, not attributable to an account", + OffhostBackupKeyUse: + "per encryption key, when the nightly off-host run last wrote an object needing it; says nothing about any account", + OffhostPurgeRequest: + "the deletions an account deletion or wipe asked of the off-host bucket; written by the wipe itself and removed once the bucket holds nothing of the account", CycleSymptomCategory: "seeded catalogue shared by every account; a user's custom symptoms live on CycleSymptom", IllnessSymptom: "seeded symptom catalogue shared by every account", diff --git a/src/lib/export/store-backup-blob.ts b/src/lib/export/store-backup-blob.ts index 36b3b2494..92d5cd785 100644 --- a/src/lib/export/store-backup-blob.ts +++ b/src/lib/export/store-backup-blob.ts @@ -34,6 +34,7 @@ import { type BackupJsonProducer, type PackBackupOptions, } from "@/lib/export/backup-blob"; +import { createBackupKeyIdTextScanner } from "@/lib/export/backup-key-ids"; import { BackupIntegrityError, newChunkStreamId, @@ -112,6 +113,11 @@ export async function storeBackupBlob( await tx.dataBackupChunk.deleteMany({ where: { backupId: row.id } }); const streamId = newChunkStreamId(); + // The key ids the copy's content was written under, read from the JSON + // on its way into the pieces. Rotation re-seals the pieces but never + // what is inside them, so this is what tells an operator how long a + // retired key is still needed (`GET /api/admin/encryption/status`). + const keyScanner = createBackupKeyIdTextScanner(); const { chunks, bytes } = await packBackupChunks( async (sealed, seq) => { await tx.dataBackupChunk.create({ @@ -120,7 +126,11 @@ export async function storeBackupBlob( }); }, streamId, - producer, + (write) => + producer(async (chunk) => { + keyScanner.feed(chunk); + await write(chunk); + }), options, ); @@ -135,6 +145,8 @@ export async function storeBackupBlob( ...(input.ownerAfterRead ? { userId: input.ownerAfterRead() } : {}), chunkCount: chunks, chunkStreamId: streamId, + innerKeyIds: keyScanner.keyIds(), + innerKeyIdsRecorded: true, createdAt: new Date(), }, }); diff --git a/src/lib/jobs/__tests__/offhost-backup.test.ts b/src/lib/jobs/__tests__/offhost-backup.test.ts index a9d07113c..57c9c07bc 100644 --- a/src/lib/jobs/__tests__/offhost-backup.test.ts +++ b/src/lib/jobs/__tests__/offhost-backup.test.ts @@ -32,6 +32,14 @@ import { const ENC_KEY = "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"; +/** The purge ledger and key-use rows the run touches after each upload. */ +function purgeLedgerMocks() { + return { + offhostPurgeRequest: { count: vi.fn().mockResolvedValue(0) }, + offhostBackupKeyUse: { upsert: vi.fn() }, + }; +} + describe("offhost-backup envelope", () => { it("encrypts and decrypts JSON round-trip", () => { const key = Buffer.from(ENC_KEY, "hex"); @@ -184,8 +192,10 @@ describe("runOffhostBackup", () => { s3.store.set("2020-01-01/user-old.json.enc", Buffer.from([0])); const prisma = { + ...purgeLedgerMocks(), offhostBackupState: { upsert: vi.fn() }, user: { + count: vi.fn().mockResolvedValue(1), findMany: vi.fn().mockResolvedValue([{ id: "u1" }, { id: "u2" }]), }, measurement: { findMany: vi.fn().mockResolvedValue([]) }, @@ -248,11 +258,43 @@ describe("runOffhostBackup", () => { expect(first.create.lastSuccessAt).toBeInstanceOf(Date); }); + it("takes back the copy of an account deleted while it was written", async () => { + const s3 = makeS3Mock(); + const ledger = purgeLedgerMocks(); + const prisma = { + ...ledger, + offhostBackupState: { upsert: vi.fn() }, + user: { + // u1 is gone by the time its upload lands; u2 is not. + count: vi.fn(async ({ where }: { where: { id: string } }) => + where.id === "u1" ? 0 : 1, + ), + findMany: vi.fn().mockResolvedValue([{ id: "u1" }, { id: "u2" }]), + }, + }; + const report = await runOffhostBackup( + // eslint-disable-next-line @typescript-eslint/no-explicit-any + prisma as any, + s3, + new Date("2026-05-08T03:00:00Z"), + ); + expect(s3.store.has("2026-05-08/user-u1.json.enc")).toBe(false); + expect(s3.store.has("2026-05-08/user-u2.json.enc")).toBe(true); + expect(s3.deleteObject).toHaveBeenCalledWith("2026-05-08/user-u1.json.enc"); + expect(report.uploaded).toBe(1); + // No ledger row for an account that no longer exists. + expect(prisma.offhostBackupState.upsert).toHaveBeenCalledTimes(1); + }); + it("uploads the canonical builder output without reshaping it", async () => { const s3 = makeS3Mock(); const prisma = { + ...purgeLedgerMocks(), offhostBackupState: { upsert: vi.fn() }, - user: { findMany: vi.fn().mockResolvedValue([{ id: "u1" }]) }, + user: { + count: vi.fn().mockResolvedValue(1), + findMany: vi.fn().mockResolvedValue([{ id: "u1" }]), + }, }; const canonicalPayload = { schemaVersion: BACKUP_SCHEMA_VERSION, @@ -310,8 +352,10 @@ describe("runOffhostBackup", () => { const s3 = makeS3Mock(); const upsert = vi.fn(); const prisma = { + ...purgeLedgerMocks(), offhostBackupState: { upsert }, user: { + count: vi.fn().mockResolvedValue(1), findMany: vi.fn().mockResolvedValue([{ id: "u1" }, { id: "u2" }]), }, }; @@ -357,10 +401,14 @@ describe("runOffhostBackup", () => { it("still reports a landed object as uploaded when the ledger write fails", async () => { const s3 = makeS3Mock(); const prisma = { + ...purgeLedgerMocks(), offhostBackupState: { upsert: vi.fn().mockRejectedValue(new Error("pool timeout")), }, - user: { findMany: vi.fn().mockResolvedValue([{ id: "u1" }]) }, + user: { + count: vi.fn().mockResolvedValue(1), + findMany: vi.fn().mockResolvedValue([{ id: "u1" }]), + }, }; const report = await runOffhostBackup( @@ -382,8 +430,12 @@ describe("runOffhostBackup", () => { it("refuses an account whose object outgrows one multipart upload", async () => { const s3 = makeS3Mock(); const prisma = { + ...purgeLedgerMocks(), offhostBackupState: { upsert: vi.fn() }, - user: { findMany: vi.fn().mockResolvedValue([{ id: "u1" }]) }, + user: { + count: vi.fn().mockResolvedValue(1), + findMany: vi.fn().mockResolvedValue([{ id: "u1" }]), + }, }; mocks.buildFullBackupPayload.mockResolvedValue({ payload: { @@ -419,8 +471,12 @@ describe("runOffhostBackup", () => { it("reports the largest object it wrote", async () => { const s3 = makeS3Mock(); const prisma = { + ...purgeLedgerMocks(), offhostBackupState: { upsert: vi.fn() }, - user: { findMany: vi.fn().mockResolvedValue([{ id: "u1" }]) }, + user: { + count: vi.fn().mockResolvedValue(1), + findMany: vi.fn().mockResolvedValue([{ id: "u1" }]), + }, }; const report = await runOffhostBackup( prisma as never, diff --git a/src/lib/jobs/__tests__/offhost-purge.test.ts b/src/lib/jobs/__tests__/offhost-purge.test.ts new file mode 100644 index 000000000..e577bbea5 --- /dev/null +++ b/src/lib/jobs/__tests__/offhost-purge.test.ts @@ -0,0 +1,205 @@ +/** + * Off-host copies leave the bucket with the account. + * + * The bucket is a map behind the same `S3Like` boundary the real client + * implements; the request ledger is an in-memory table. Mutations that must + * turn this red: drop the `deleteObject` call (the objects stay), remove the + * request on failure too (the refusal case loses the request), or match on the + * key prefix instead of the full account pattern (another account's objects, + * or the health-check probe, go too). + */ +import { beforeEach, describe, expect, it, vi } from "vitest"; + +vi.mock("@/lib/auth/audit", () => ({ auditLog: vi.fn() })); + +import { auditLog } from "@/lib/auth/audit"; +import { + processOffhostPurges, + PURGE_ALL_SUBJECTS, + requestOffhostPurge, +} from "../offhost-purge"; + +interface Row { + id: string; + subjectId: string; + reason: string; + requestedAt: Date; + attempts: number; + lastAttemptAt: Date | null; + lastFailure: string | null; +} + +function makeLedger(existingUsers: string[] = []) { + const rows: Row[] = []; + let seq = 0; + const prisma = { + offhostPurgeRequest: { + create: vi.fn( + async ({ data }: { data: { subjectId: string; reason: string } }) => { + const row: Row = { + id: `r${++seq}`, + subjectId: data.subjectId, + reason: data.reason, + requestedAt: new Date(Date.now() + seq), + attempts: 0, + lastAttemptAt: null, + lastFailure: null, + }; + rows.push(row); + return { id: row.id }; + }, + ), + findMany: vi.fn(async () => + [...rows].sort((a, b) => +a.requestedAt - +b.requestedAt), + ), + delete: vi.fn(async ({ where }: { where: { id: string } }) => { + rows.splice( + rows.findIndex((r) => r.id === where.id), + 1, + ); + }), + updateMany: vi.fn( + async ({ + where, + data, + }: { + where: { id: { in: string[] } }; + data: { lastAttemptAt: Date; lastFailure: string }; + }) => { + for (const row of rows) { + if (!where.id.in.includes(row.id)) continue; + row.attempts += 1; + row.lastAttemptAt = data.lastAttemptAt; + row.lastFailure = data.lastFailure; + } + }, + ), + }, + user: { + count: vi.fn(async ({ where }: { where: { id: string } }) => + existingUsers.includes(where.id) ? 1 : 0, + ), + }, + }; + return { rows, prisma }; +} + +function makeBucket(keys: string[]) { + const store = new Set(keys); + return { + store, + putObject: vi.fn(), + putStream: vi.fn(), + getObject: vi.fn(), + headObject: vi.fn(), + listObjects: vi.fn(async (prefix: string) => + [...store].filter((k) => k.startsWith(prefix)).map((key) => ({ key })), + ), + deleteObject: vi.fn(async (key: string) => { + store.delete(key); + }), + }; +} + +const BUCKET = [ + "2026-09-01/user-gone.json.enc", + "2026-09-02/user-gone.json.enc", + "2026-09-02/user-gone2.json.enc", + "2026-09-02/user-stays.json.enc", + "_healthcheck/1.bin", +]; + +beforeEach(() => { + vi.unstubAllEnvs(); + vi.clearAllMocks(); + vi.stubEnv("BACKUP_S3_ENDPOINT", "https://r2.example"); + vi.stubEnv("BACKUP_S3_BUCKET", "hl-backups"); + vi.stubEnv("BACKUP_S3_ACCESS_KEY", "AKIA"); + vi.stubEnv("BACKUP_S3_SECRET_KEY", "secret"); + vi.stubEnv("BACKUP_ENCRYPTION_KEY", "ab".repeat(32)); +}); + +describe("off-host purge", () => { + it("deletes every copy of the account, and nothing else", async () => { + const { rows, prisma } = makeLedger(); + await requestOffhostPurge(prisma as never, "gone", "account_deleted"); + const bucket = makeBucket(BUCKET); + + const report = await processOffhostPurges(prisma as never, bucket); + + expect([...bucket.store].sort()).toEqual([ + "2026-09-02/user-gone2.json.enc", + "2026-09-02/user-stays.json.enc", + "_healthcheck/1.bin", + ]); + expect(report).toMatchObject({ + completed: 1, + failed: 0, + objectsDeleted: 2, + }); + expect(rows).toHaveLength(0); + // The account is gone, and so is its audit trail: the receipt carries the + // reason and the count, not the account. + expect(auditLog).toHaveBeenCalledWith( + "offhost.backup.purged", + expect.objectContaining({ + userId: null, + details: expect.objectContaining({ + reason: "account_deleted", + objectsDeleted: 2, + }), + }), + ); + }); + + it("keeps the request, with the bucket's answer, when a delete is refused", async () => { + const { rows, prisma } = makeLedger(["gone"]); + await requestOffhostPurge(prisma as never, "gone", "data_wiped"); + const bucket = makeBucket(BUCKET); + bucket.deleteObject.mockRejectedValueOnce(new Error("AccessDenied")); + + const report = await processOffhostPurges(prisma as never, bucket); + + expect(report).toMatchObject({ completed: 0, failed: 1 }); + expect(rows).toHaveLength(1); + expect(rows[0]).toMatchObject({ attempts: 1, lastFailure: "AccessDenied" }); + + // The next run finishes it, and the account that still exists keeps the + // receipt. + const again = await processOffhostPurges(prisma as never, bucket); + expect(again.completed).toBe(1); + expect(rows).toHaveLength(0); + expect(auditLog).toHaveBeenCalledWith( + "offhost.backup.purged", + expect.objectContaining({ userId: "gone" }), + ); + }); + + it("clears every account object for the admin wipe", async () => { + const { prisma } = makeLedger(); + await requestOffhostPurge( + prisma as never, + PURGE_ALL_SUBJECTS, + "admin_wipe", + ); + const bucket = makeBucket(BUCKET); + await processOffhostPurges(prisma as never, bucket); + expect([...bucket.store]).toEqual(["_healthcheck/1.bin"]); + }); + + it("records nothing on a host without off-host backup", async () => { + vi.stubEnv("BACKUP_S3_ENDPOINT", ""); + const { rows, prisma } = makeLedger(); + expect( + await requestOffhostPurge(prisma as never, "gone", "account_deleted"), + ).toBe(false); + expect(rows).toHaveLength(0); + }); + + it("does not list the bucket when nothing is pending", async () => { + const { prisma } = makeLedger(); + const bucket = makeBucket(BUCKET); + await processOffhostPurges(prisma as never, bucket); + expect(bucket.listObjects).not.toHaveBeenCalled(); + }); +}); diff --git a/src/lib/jobs/job-outcome.ts b/src/lib/jobs/job-outcome.ts index 3df4416ce..78dc759b8 100644 --- a/src/lib/jobs/job-outcome.ts +++ b/src/lib/jobs/job-outcome.ts @@ -144,6 +144,10 @@ export const JOB_FACT_ALLOWLIST: ReadonlySet = new Set([ "offhost_backup_oversized", "offhost_backup_total_users", "offhost_backup_uploaded", + "offhost_purge_completed", + "offhost_purge_failed", + "offhost_purge_objects_deleted", + "offhost_purge_pending", "outcome", "passes_failed", "per_sample_rows_deleted", diff --git a/src/lib/jobs/offhost-backup.ts b/src/lib/jobs/offhost-backup.ts index 13504ff67..9a21b15f2 100644 --- a/src/lib/jobs/offhost-backup.ts +++ b/src/lib/jobs/offhost-backup.ts @@ -27,6 +27,7 @@ import { Readable, Transform } from "node:stream"; import { createGunzip, createGzip, gunzipSync, gzipSync } from "node:zlib"; import type { PrismaClient } from "@/generated/prisma/client"; import { createRawStreamEncryptor, decryptRawStream } from "@/lib/crypto"; +import { createBackupKeyIdTextScanner } from "@/lib/export/backup-key-ids"; import { streamFullBackupJson } from "@/lib/export/full-backup-stream"; import { annotate, getEvent } from "@/lib/logging/context"; @@ -489,6 +490,24 @@ export interface S3Like { prefix: string, ): Promise>; deleteObject(key: string): Promise; + /** + * The bucket's lifecycle rule, as far as the credential may read it. + * Optional so a test double that has no opinion leaves it out. + */ + getLifecycle?(): Promise; +} + +/** + * Whether the bucket retires old copies by itself. + * + * `configured` names the shortest expiry of an enabled rule; `missing` means + * the bucket answered that it has no rule; `unknown` means it would not say + * (a credential without `s3:GetLifecycleConfiguration`, a target that does not + * implement the call, a network failure) and says nothing either way. + */ +export interface OffhostLifecycle { + state: "configured" | "missing" | "unknown"; + expirationDays: number | null; } export async function getS3Client(cfg: OffhostBackupConfig): Promise { @@ -601,9 +620,78 @@ export async function getS3Client(cfg: OffhostBackupConfig): Promise { new mod.DeleteObjectCommand({ Bucket: cfg.bucket, Key: key }), ); }, + getLifecycle: async () => { + try { + const out = await client.send( + new mod.GetBucketLifecycleConfigurationCommand({ + Bucket: cfg.bucket, + }), + { abortSignal: AbortSignal.timeout(5_000) }, + ); + const days = (out.Rules ?? []) + .filter((rule) => rule.Status === "Enabled") + .map((rule) => rule.Expiration?.Days) + .filter((d): d is number => typeof d === "number" && d > 0); + return days.length > 0 + ? { state: "configured", expirationDays: Math.min(...days) } + : { state: "missing", expirationDays: null }; + } catch (err) { + const name = (err as { name?: string; Code?: string }).name; + const code = (err as { Code?: string }).Code; + if ( + name === "NoSuchLifecycleConfiguration" || + code === "NoSuchLifecycleConfiguration" + ) { + return { state: "missing", expirationDays: null }; + } + return { state: "unknown", expirationDays: null }; + } + }, }; } +const LIFECYCLE_CACHE_MS = 10 * 60 * 1000; +let lifecycleCache: { at: number; value: OffhostLifecycle } | null = null; + +/** + * The bucket's lifecycle rule for the admin card, cached for ten minutes so a + * page render does not become a bucket call. Never throws. + */ +export async function probeOffhostLifecycle( + s3Override?: S3Like, + now: number = Date.now(), +): Promise { + if ( + !s3Override && + lifecycleCache && + now - lifecycleCache.at < LIFECYCLE_CACHE_MS + ) { + return lifecycleCache.value; + } + const cfg = loadOffhostConfigSafe(); + if (!cfg) return { state: "unknown", expirationDays: null }; + let value: OffhostLifecycle; + try { + const s3 = s3Override ?? (await getS3Client(cfg)); + value = s3.getLifecycle + ? await s3.getLifecycle() + : { state: "unknown", expirationDays: null }; + } catch { + value = { state: "unknown", expirationDays: null }; + } + if (!s3Override) lifecycleCache = { at: now, value }; + return value; +} + +/** `loadOffhostConfig`, answering null for a malformed key instead of throwing. */ +function loadOffhostConfigSafe(): OffhostBackupConfig | null { + try { + return loadOffhostConfig(); + } catch { + return null; + } +} + interface BackupRunReport { config: { endpoint: string; bucket: string; region: string }; uploaded: number; @@ -642,6 +730,30 @@ export interface RunOffhostBackupOptions extends UploadBackupOptions { shouldStop?: () => boolean; } +/** + * Note, per key id, that an object needing it went into the bucket now. Never + * fails the account: the object is in the bucket whatever this row says. + */ +async function recordOffhostKeyUse( + prisma: PrismaClient, + keyIds: readonly string[], + at: Date, +): Promise { + for (const keyId of keyIds) { + try { + await prisma.offhostBackupKeyUse.upsert({ + where: { keyId }, + create: { keyId, firstWrittenAt: at, lastWrittenAt: at }, + update: { lastWrittenAt: at }, + }); + } catch (err) { + getEvent()?.addWarning( + `offhost-backup key-use write failed: ${(err as Error).message?.slice(0, 200)}`, + ); + } + } +} + export async function runOffhostBackup( prisma: PrismaClient, s3Override?: S3Like, @@ -689,24 +801,54 @@ export async function runOffhostBackup( break; } let objectBytes: number | null = null; + const objectKey = `${dateKey}/user-${user.id}.json.enc`; + const accountStartedAt = new Date(); + // The key ids the object's content needs, read as it is written: the + // admin encryption view uses them to say how long a retired key is still + // needed for what is in the bucket. + const keyScanner = createBackupKeyIdTextScanner(); try { objectBytes = await uploadEncryptedBackup( s3, - `${dateKey}/user-${user.id}.json.enc`, + objectKey, cfg.encryptionKey, // The same writer the weekly in-database pass uses. The payload // builder was always shared; everything after it was not, which is why // this job kept dying on a record the weekly one had learned to // survive. (write) => - streamFullBackupJson(prisma, user.id, write, { - purpose: "disaster-recovery", - exportedAt: now, - }), + streamFullBackupJson( + prisma, + user.id, + async (chunk) => { + keyScanner.feed(chunk); + await write(chunk); + }, + { + purpose: "disaster-recovery", + exportedAt: now, + }, + ), options, ); + // The account was deleted or wiped while its copy was being written. + // The purge that request started may already have run, so the copy + // this run just put there is removed here rather than left behind. + const purgedMeanwhile = + (await prisma.offhostPurgeRequest.count({ + where: { + subjectId: { in: [user.id, "*"] }, + requestedAt: { gte: accountStartedAt }, + }, + })) > 0 || (await prisma.user.count({ where: { id: user.id } })) === 0; + if (purgedMeanwhile) { + await s3.deleteObject(objectKey); + objectBytes = null; + continue; + } largestObjectBytes = Math.max(largestObjectBytes, objectBytes); uploaded++; + await recordOffhostKeyUse(prisma, keyScanner.keyIds(), new Date()); } catch (err) { failed++; if (err instanceof OffhostBackupTooLargeError) oversized++; diff --git a/src/lib/jobs/offhost-purge.ts b/src/lib/jobs/offhost-purge.ts new file mode 100644 index 000000000..91adad732 --- /dev/null +++ b/src/lib/jobs/offhost-purge.ts @@ -0,0 +1,270 @@ +/** + * Off-host copies leave the bucket with the account. + * + * The nightly job writes `/user-.json.enc` for every account, + * and until v1.39.3 nothing ever removed one: deleting an account, or wiping + * its data, took the rows in this database and left every nightly copy of the + * record in the operator's bucket for as long as its lifecycle rule kept them, + * or forever where no rule was set. A confirmation that says "permanently + * delete" has to mean the off-host copies too. + * + * How. The deletion writes an `OffhostPurgeRequest` in its own transaction, so + * the request exists exactly when the deletion committed. This job then lists + * the bucket once, deletes every object of every pending account, and removes + * the request when nothing of the account is left. A request the bucket + * refuses (a credential without DeleteObject, an outage) stays, counts its + * attempts and keeps the bucket's answer, which the admin off-host card shows; + * the job runs again every night and after every nightly upload, so a late + * copy the upload wrote for an account deleted while it ran is caught too. + * + * `subjectId` "*" is the admin wipe of every account: every account object + * goes. The health-check probes and anything else in the bucket that is not + * an account object is never touched. + */ +import type { Job } from "pg-boss"; + +import type { Prisma, PrismaClient } from "@/generated/prisma/client"; +import { auditLog } from "@/lib/auth/audit"; +import { getGlobalBoss } from "@/lib/jobs/boss-instance"; +import { jobDone, jobFailed, type JobOutcome } from "@/lib/jobs/job-outcome"; +import { + getS3Client, + loadOffhostConfig, + offhostBackupConfigured, + type S3Like, +} from "@/lib/jobs/offhost-backup"; +import { reportWorkerError } from "@/lib/jobs/report-worker-error"; +import { getWorkerPrisma } from "@/lib/jobs/reminder/shared"; +import { withBackgroundEvent } from "@/lib/logging/background"; + +export const OFFHOST_PURGE_QUEUE = "offhost-backup-purge"; +/** 03:40, after the nightly upload (02:30) has had its hour. */ +export const OFFHOST_PURGE_CRON = "40 3 * * *"; + +/** Every subject, for the admin wipe of every account. */ +export const PURGE_ALL_SUBJECTS = "*"; + +export type OffhostPurgeReason = + "account_deleted" | "data_wiped" | "managed_profile_deleted" | "admin_wipe"; + +/** An account object: `/user-.json.enc`. */ +const ACCOUNT_OBJECT = /^\d{4}-\d{2}-\d{2}\/user-(.+)\.json\.enc$/; + +/** Requests one run works through; the next run takes the rest. */ +const REQUESTS_PER_RUN = 200; + +/** Deletes one run may issue, so a huge bucket cannot run the job forever. */ +const MAX_DELETES_PER_RUN = 50_000; + +type Db = Pick; + +/** + * Record that `subjectId`'s off-host copies have to go. Call it with the + * transaction that deletes the account or wipes its data, so the request and + * the deletion commit together. A host without off-host backup configured has + * nothing in any bucket this could reach, and records nothing. + */ +export async function requestOffhostPurge( + db: Db, + subjectId: string, + reason: OffhostPurgeReason, +): Promise { + if (!offhostBackupConfigured()) return false; + await db.offhostPurgeRequest.create({ + data: { subjectId, reason }, + select: { id: true }, + }); + return true; +} + +/** + * Wake the purge job now rather than at its nightly slot. Best effort: the + * request row is what guarantees the purge, and the cron picks it up if the + * queue is not reachable from here. + */ +export async function kickOffhostPurge(): Promise { + try { + await getGlobalBoss()?.send( + OFFHOST_PURGE_QUEUE, + {}, + { + singletonKey: OFFHOST_PURGE_QUEUE, + retryLimit: 3, + retryDelay: 300, + }, + ); + } catch { + // The cron is the backstop. + } +} + +export interface OffhostPurgeReport { + pending: number; + completed: number; + failed: number; + objectsDeleted: number; +} + +function trimError(err: unknown): string { + return (err instanceof Error ? err.message : String(err)).slice(0, 300); +} + +/** + * Work through the pending requests: list the bucket once, delete every + * object of every pending subject, remove each request whose objects are all + * gone. Answers what it did. + */ +export async function processOffhostPurges( + prisma: PrismaClient, + s3Override?: S3Like, +): Promise { + const requests = await prisma.offhostPurgeRequest.findMany({ + orderBy: { requestedAt: "asc" }, + take: REQUESTS_PER_RUN, + }); + const report: OffhostPurgeReport = { + pending: requests.length, + completed: 0, + failed: 0, + objectsDeleted: 0, + }; + if (requests.length === 0) return report; + + const cfg = loadOffhostConfig(); + if (!cfg) { + // Configuration removed after the requests were written: leave them, an + // operator who puts it back gets the purge. + return report; + } + + const now = new Date(); + const markFailed = async (ids: string[], err: unknown) => { + await prisma.offhostPurgeRequest.updateMany({ + where: { id: { in: ids } }, + data: { + attempts: { increment: 1 }, + lastAttemptAt: now, + lastFailure: trimError(err), + }, + }); + report.failed += ids.length; + }; + + let objects: Array<{ key: string }>; + let s3: S3Like; + try { + s3 = s3Override ?? (await getS3Client(cfg)); + objects = await s3.listObjects(""); + } catch (err) { + await markFailed( + requests.map((r) => r.id), + err, + ); + return report; + } + + const everything = requests.some((r) => r.subjectId === PURGE_ALL_SUBJECTS); + const subjects = new Set(requests.map((r) => r.subjectId)); + const bySubject = new Map(); + for (const { key } of objects) { + const match = ACCOUNT_OBJECT.exec(key); + if (!match) continue; + const subject = match[1]; + if (!everything && !subjects.has(subject)) continue; + const list = bySubject.get(subject) ?? []; + list.push(key); + bySubject.set(subject, list); + } + + // One subject at a time, so a refusal part-way fails only the subjects it + // reached, and a request is removed only when its own objects are gone. + const failedSubjects = new Map(); + let deletes = 0; + for (const [subject, keys] of bySubject) { + for (const key of keys) { + if (deletes >= MAX_DELETES_PER_RUN) { + failedSubjects.set( + subject, + new Error("Deletion budget for one run reached; continues next run"), + ); + break; + } + try { + await s3.deleteObject(key); + deletes++; + report.objectsDeleted++; + } catch (err) { + failedSubjects.set(subject, err); + break; + } + } + } + + for (const request of requests) { + const failure = + request.subjectId === PURGE_ALL_SUBJECTS + ? failedSubjects.size > 0 + ? [...failedSubjects.values()][0] + : null + : failedSubjects.get(request.subjectId); + if (failure) { + await markFailed([request.id], failure); + continue; + } + const deleted = + request.subjectId === PURGE_ALL_SUBJECTS + ? [...bySubject.values()].reduce((sum, keys) => sum + keys.length, 0) + : (bySubject.get(request.subjectId)?.length ?? 0); + await prisma.offhostPurgeRequest.delete({ where: { id: request.id } }); + report.completed++; + // The account a deleted subject named is gone, and so is its audit + // history; the receipt names the reason and the count, not the account. + // A wiped account still exists and keeps the receipt as its own. + const accountRemains = + request.reason === "data_wiped" && + (await prisma.user.count({ where: { id: request.subjectId } })) > 0; + await auditLog("offhost.backup.purged", { + userId: accountRemains ? request.subjectId : null, + actorUserId: null, + details: { + reason: request.reason, + objectsDeleted: deleted, + attempts: request.attempts + 1, + }, + }); + } + return report; +} + +export async function handleOffhostPurge( + jobs: Job[], +): Promise { + void jobs; + return withBackgroundEvent("job.offhost_purge", async (evt) => { + try { + const report = await processOffhostPurges(getWorkerPrisma()); + evt.addMeta("offhost_purge_pending", report.pending); + evt.addMeta("offhost_purge_completed", report.completed); + evt.addMeta("offhost_purge_failed", report.failed); + evt.addMeta("offhost_purge_objects_deleted", report.objectsDeleted); + if (report.failed > 0) { + await reportWorkerError( + OFFHOST_PURGE_QUEUE, + new Error( + `${report.failed} off-host deletion request(s) could not be completed; they stay pending and are retried nightly. Check that the bucket credential allows DeleteObject.`, + ), + ); + } + return jobDone({ + offhost_purge_pending: report.pending, + offhost_purge_completed: report.completed, + offhost_purge_failed: report.failed, + offhost_purge_objects_deleted: report.objectsDeleted, + }); + } catch (err) { + evt.addWarning(`offhost-purge failed: ${err}`); + await reportWorkerError(OFFHOST_PURGE_QUEUE, err); + return jobFailed("offhost purge failed", err); + } + }); +} diff --git a/src/lib/jobs/reminder/register-maintenance.ts b/src/lib/jobs/reminder/register-maintenance.ts index 0aa69e355..149e91d45 100644 --- a/src/lib/jobs/reminder/register-maintenance.ts +++ b/src/lib/jobs/reminder/register-maintenance.ts @@ -67,6 +67,11 @@ import { enqueueBootTimeIntakeSlotDedup, type IntakeSlotDedupPayload, } from "@/lib/medications/intake-slot-dedup"; +import { + handleOffhostPurge, + OFFHOST_PURGE_CRON, + OFFHOST_PURGE_QUEUE, +} from "@/lib/jobs/offhost-purge"; import { handleRestoreDrill, RESTORE_DRILL_CRON, @@ -353,6 +358,10 @@ const allQueues = [ STEP_UP_ELEVATION_CLEANUP_QUEUE, OFFHOST_BACKUP_QUEUE, RESTORE_DRILL_QUEUE, + // v1.39.3 — deleting an account or wiping its data takes its off-host + // copies out of the bucket. Without this entry the requests would be + // written and never worked. + OFFHOST_PURGE_QUEUE, // v1.39.1 — an admin's restore of a stored backup, run off the request, and // the boot sweep that re-queues one a stopped worker left running. Without // this entry every restore request would queue a job nobody works. @@ -505,6 +514,9 @@ const schedules: ScheduleEntry[] = [ ], [OFFHOST_BACKUP_QUEUE, OFFHOST_BACKUP_CRON], [RESTORE_DRILL_QUEUE, RESTORE_DRILL_CRON], + // Nightly backstop for the purge the deletion kicks straight away: a + // request the bucket refused, or one written while the queue was down. + [OFFHOST_PURGE_QUEUE, OFFHOST_PURGE_CRON, cronIsTheRetry], [HOST_METRIC_QUEUE, HOST_METRIC_CRON, cronIsTheRetry], [FEEDBACK_AGGREGATOR_QUEUE, FEEDBACK_AGGREGATOR_CRON, cronIsTheRetry], // v1.4.37 — hourly geo backfill. The helper is idempotent + capped @@ -747,6 +759,12 @@ export async function registerMaintenanceQueues( { localConcurrency: 1 }, handleRestoreDrill, ); + await createAndWork( + boss, + OFFHOST_PURGE_QUEUE, + { localConcurrency: 1 }, + handleOffhostPurge, + ); // One restore at a time per worker: each holds a transaction over a whole // account and a batch of readings in memory, and two at once on a small host // is how a restore that fits alone would run out of heap. diff --git a/src/lib/jobs/subsystem-surface.ts b/src/lib/jobs/subsystem-surface.ts index bbfeffc72..cdf8b17d4 100644 --- a/src/lib/jobs/subsystem-surface.ts +++ b/src/lib/jobs/subsystem-surface.ts @@ -101,6 +101,7 @@ const subsystemSurface = { "step-up-elevation-cleanup": { audience: "system" }, "data-backup-offhost": { audience: "system" }, "data-restore-drill": { audience: "system" }, + "offhost-backup-purge": { audience: "system" }, "backup-restore": { audience: "system" }, "host-metric-sample": { audience: "system" }, "feedback-aggregator": { audience: "system" }, diff --git a/src/lib/managed-profiles/lifecycle.ts b/src/lib/managed-profiles/lifecycle.ts index 44ddac8ac..1160825e5 100644 --- a/src/lib/managed-profiles/lifecycle.ts +++ b/src/lib/managed-profiles/lifecycle.ts @@ -1,3 +1,7 @@ +import { + kickOffhostPurge, + requestOffhostPurge, +} from "@/lib/jobs/offhost-purge"; import { prisma } from "@/lib/db"; import { auditLog } from "@/lib/auth/audit"; import type { Prisma } from "@/generated/prisma/client"; @@ -209,9 +213,11 @@ export async function deleteManagedProfile(input: { details: { profileId: profile.id }, client: tx, }); + await requestOffhostPurge(tx, profile.id, "managed_profile_deleted"); await tx.user.delete({ where: { id: profile.id } }); }); }); + await kickOffhostPurge(); } /** diff --git a/src/lib/openapi/error-codes.ts b/src/lib/openapi/error-codes.ts index 92602141d..bf1eaa48c 100644 --- a/src/lib/openapi/error-codes.ts +++ b/src/lib/openapi/error-codes.ts @@ -66,6 +66,8 @@ export const ERROR_CODE_CATALOGUE: Readonly> = "auth.token.invalid", ], backup: [ + "backup.foreign_reference", + "backup.key.missing", "backup.payload.undecryptable", "backup.restore.active", "backup.section.missing", diff --git a/src/types/backups.ts b/src/types/backups.ts index 635b65c27..fb7b1c18d 100644 --- a/src/types/backups.ts +++ b/src/types/backups.ts @@ -58,6 +58,25 @@ export interface OffhostBackupOverview { periodHours: number; /** One row per account. Empty when off-host backup is not configured. */ rows: OffhostAccountRow[]; + /** + * Deleted or wiped accounts whose copies are still in the bucket. A count + * above zero with an error means the bucket refused the deletion (most + * often a credential without DeleteObject); the job retries nightly. + */ + pendingDeletions: { + count: number; + oldestRequestedAt: string | null; + lastFailure: string | null; + }; + /** + * Whether the bucket retires old copies by itself. Read from the bucket + * (`GetBucketLifecycleConfiguration`), cached for ten minutes; `unknown` + * when the credential may not read it. + */ + lifecycle: { + state: "configured" | "missing" | "unknown"; + expirationDays: number | null; + }; } export interface BackupsList { diff --git a/tests/integration/backup-key-needs.test.ts b/tests/integration/backup-key-needs.test.ts new file mode 100644 index 000000000..206c38e97 --- /dev/null +++ b/tests/integration/backup-key-needs.test.ts @@ -0,0 +1,126 @@ +/** + * A stored backup records the keys its content needs, and the encryption view + * says a retired key is still needed while any backup lists it. + * + * Both ends and the pipe: the copy is written by the real `storeBackupBlob`, + * the key ids are read from the JSON on its way into the pieces, and the + * admin status route reads them back. The case the view used to get wrong is + * the one asserted: every row rotated, the badge said the old key was safe to + * drop, and a backup still needed it. + * + * Mutations that must turn this red: drop the scanner from `storeBackupBlob` + * (nothing is recorded, the view has no key to name), or leave + * `retiredKeysStillNeeded` out of `safeToDropRetiredKeys` (the badge says + * safe again). + */ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +import { _resetCryptoCacheForTests, encrypt } from "@/lib/crypto"; + +import { cookieJar, headerJar } from "./mock-next-headers"; +import { getPrismaClient, truncateAllTables } from "./setup"; + +vi.mock("next/headers", async () => { + const { cookieJar, headerJar } = await import("./mock-next-headers"); + return { + headers: vi.fn(async () => ({ + get: (name: string) => headerJar.get(name.toLowerCase()) ?? null, + })), + cookies: vi.fn(async () => ({ + get: (name: string) => { + const value = cookieJar.get(name); + return value ? { name, value } : undefined; + }, + set: (name: string, value: string) => { + cookieJar.set(name, value); + }, + delete: (name: string) => { + cookieJar.delete(name); + }, + })), + }; +}); + +vi.mock("@/lib/db-compat", () => ({ + ensureDbCompatibility: vi.fn().mockResolvedValue(undefined), +})); + +const saved = { + key: process.env.ENCRYPTION_KEY, + keys: process.env.ENCRYPTION_KEYS, + active: process.env.ENCRYPTION_ACTIVE_KEY_ID, +}; + +function useKeys(active: string) { + process.env.ENCRYPTION_KEY = ""; + process.env.ENCRYPTION_KEYS = JSON.stringify({ + old: "11".repeat(32), + cur: "22".repeat(32), + }); + process.env.ENCRYPTION_ACTIVE_KEY_ID = active; + _resetCryptoCacheForTests(); +} + +beforeEach(async () => { + await truncateAllTables(getPrismaClient()); + cookieJar.clear(); + headerJar.clear(); +}); + +afterEach(() => { + process.env.ENCRYPTION_KEY = saved.key ?? ""; + process.env.ENCRYPTION_KEYS = saved.keys ?? ""; + process.env.ENCRYPTION_ACTIVE_KEY_ID = saved.active ?? ""; + _resetCryptoCacheForTests(); +}); + +describe("the keys stored backups still need", () => { + it("names a retired key a backup's content was written under", async () => { + const prisma = getPrismaClient(); + const admin = await prisma.user.create({ + data: { username: "keys-admin", email: "k@example.test", role: "ADMIN" }, + }); + const session = await prisma.session.create({ + data: { userId: admin.id, expiresAt: new Date(Date.now() + 60_000) }, + }); + cookieJar.set("healthlog_session", session.id); + + const { storeBackupBlob } = await import("@/lib/export/store-backup-blob"); + // A copy taken before the rotation: its note was written under "old". + useKeys("old"); + const note = Buffer.from(encrypt("before"), "utf8").toString("base64"); + // The copy itself is stored after the rotation, so its envelope is + // sealed under "cur" and only its content needs "old". + useKeys("cur"); + await storeBackupBlob( + prisma, + { userId: admin.id, type: "MANUAL_UPLOAD_KEYS" }, + async (write) => { + await write('{"userId":"x","measurements":[{"notesEncrypted":'); + await write(`"${note}"}]}`); + }, + ); + const row = await prisma.dataBackup.findFirstOrThrow({ + where: { type: "MANUAL_UPLOAD_KEYS" }, + }); + expect(row.innerKeyIds).toEqual(["old"]); + expect(row.innerKeyIdsRecorded).toBe(true); + + const { GET } = await import("@/app/api/admin/encryption/status/route"); + const res = await GET(); + expect(res.status).toBe(200); + const body = (await res.json()).data as { + rotationComplete: boolean; + safeToDropRetiredKeys: boolean; + backups: { + stored: Array<{ keyId: string; copies: number }>; + retiredKeysStillNeeded: string[]; + }; + }; + expect(body.backups.stored).toEqual([ + expect.objectContaining({ keyId: "old", copies: 1 }), + ]); + expect(body.backups.retiredKeysStillNeeded).toEqual(["old"]); + expect(body.safeToDropRetiredKeys).toBe(false); + }); +}); diff --git a/tests/integration/offhost-purge-on-deletion.test.ts b/tests/integration/offhost-purge-on-deletion.test.ts new file mode 100644 index 000000000..9c96deb75 --- /dev/null +++ b/tests/integration/offhost-purge-on-deletion.test.ts @@ -0,0 +1,157 @@ +/** + * Deleting an account, or wiping its data, takes its off-host copies out of + * the bucket. + * + * The bucket is replaced at the client boundary (`getS3Client`) by an + * in-memory map; everything above it is real: the route, its transaction, the + * request row it commits and the purge job that works it. Mutations that must + * turn this red: drop `requestOffhostPurge` from either route (no request is + * written and the objects stay), or write the request outside the deletion's + * transaction and let it fail (the deletion commits without one). + */ +import { NextRequest } from "next/server"; +import { beforeEach, describe, expect, it, vi } from "vitest"; + +import { cookieJar, headerJar } from "./mock-next-headers"; +import { getPrismaClient, truncateAllTables } from "./setup"; + +const bucket = vi.hoisted(() => new Set()); + +vi.mock("@/lib/jobs/offhost-backup", async (importOriginal) => { + const actual = + await importOriginal(); + return { + ...actual, + getS3Client: vi.fn(async () => ({ + putObject: vi.fn(), + putStream: vi.fn(), + getObject: vi.fn(), + headObject: vi.fn(), + listObjects: vi.fn(async (prefix: string) => + [...bucket].filter((k) => k.startsWith(prefix)).map((key) => ({ key })), + ), + deleteObject: vi.fn(async (key: string) => { + bucket.delete(key); + }), + })), + }; +}); + +vi.mock("next/headers", async () => { + const { cookieJar, headerJar } = await import("./mock-next-headers"); + return { + headers: vi.fn(async () => ({ + get: (name: string) => headerJar.get(name.toLowerCase()) ?? null, + })), + cookies: vi.fn(async () => ({ + get: (name: string) => { + const value = cookieJar.get(name); + return value ? { name, value } : undefined; + }, + set: (name: string, value: string) => { + cookieJar.set(name, value); + }, + delete: (name: string) => { + cookieJar.delete(name); + }, + })), + }; +}); + +vi.mock("@/lib/db-compat", () => ({ + ensureDbCompatibility: vi.fn().mockResolvedValue(undefined), +})); + +beforeEach(async () => { + await truncateAllTables(getPrismaClient()); + cookieJar.clear(); + headerJar.clear(); + bucket.clear(); + process.env.BACKUP_S3_ENDPOINT = "https://r2.example"; + process.env.BACKUP_S3_BUCKET = "hl-backups"; + process.env.BACKUP_S3_ACCESS_KEY = "AKIA"; + process.env.BACKUP_S3_SECRET_KEY = "secret"; + process.env.BACKUP_ENCRYPTION_KEY = "ab".repeat(32); +}); + +async function seedUserWithSession(username: string) { + const prisma = getPrismaClient(); + const user = await prisma.user.create({ + data: { username, email: `${username}@example.test` }, + }); + const session = await prisma.session.create({ + data: { userId: user.id, expiresAt: new Date(Date.now() + 3_600_000) }, + }); + cookieJar.set("healthlog_session", session.id); + return user.id; +} + +function seedBucket(userId: string, otherId: string) { + for (const day of ["2026-09-01", "2026-09-02", "2026-09-03"]) { + bucket.add(`${day}/user-${userId}.json.enc`); + bucket.add(`${day}/user-${otherId}.json.enc`); + } + bucket.add("_healthcheck/1.bin"); +} + +describe("off-host copies leave with the account", () => { + it("account deletion removes every copy of that account", async () => { + const prisma = getPrismaClient(); + const other = await prisma.user.create({ + data: { username: "neighbour", email: "neighbour@example.test" }, + }); + const userId = await seedUserWithSession("leaving"); + seedBucket(userId, other.id); + + const { DELETE } = await import("@/app/api/settings/account/route"); + const res = await DELETE( + new NextRequest("http://localhost/api/settings/account", { + method: "DELETE", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ confirm: "DELETE_ACCOUNT" }), + }), + ); + expect(res.status).toBe(200); + + const pending = await prisma.offhostPurgeRequest.findMany(); + expect(pending.map((r) => [r.subjectId, r.reason])).toEqual([ + [userId, "account_deleted"], + ]); + + const { processOffhostPurges } = await import("@/lib/jobs/offhost-purge"); + const report = await processOffhostPurges(prisma); + expect(report).toMatchObject({ completed: 1, objectsDeleted: 3 }); + expect([...bucket].filter((k) => k.includes(userId))).toEqual([]); + expect([...bucket].filter((k) => k.includes(other.id))).toHaveLength(3); + expect(bucket.has("_healthcheck/1.bin")).toBe(true); + expect(await prisma.offhostPurgeRequest.count()).toBe(0); + }); + + it("delete-all-data removes every copy and leaves the receipt", async () => { + const prisma = getPrismaClient(); + const other = await prisma.user.create({ + data: { username: "neighbour2", email: "neighbour2@example.test" }, + }); + const userId = await seedUserWithSession("wiping"); + seedBucket(userId, other.id); + + const { DELETE } = await import("@/app/api/settings/data/route"); + const res = await DELETE( + new NextRequest("http://localhost/api/settings/data", { + method: "DELETE", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ confirm: "DELETE" }), + }), + ); + expect(res.status).toBe(200); + + const { processOffhostPurges } = await import("@/lib/jobs/offhost-purge"); + await processOffhostPurges(prisma); + expect([...bucket].filter((k) => k.includes(userId))).toEqual([]); + expect([...bucket].filter((k) => k.includes(other.id))).toHaveLength(3); + const receipt = await prisma.auditLog.findFirst({ + where: { action: "offhost.backup.purged", userId }, + }); + expect(receipt).not.toBeNull(); + }); +});