mirror of
https://github.com/MBombeck/HealthLog.git
synced 2026-11-07 04:12:04 +00:00
fix(backup): take off-host copies out with the account, and say which keys backups still need
Deleting an account, deleting a managed profile, "Delete all data" and the admin wipe now write an off-host deletion request in the same transaction as the deletion. A new job (offhost-backup-purge, kicked straight away and run nightly as a backstop) lists the bucket once, deletes every <date>/user-<id>.json.enc object of each pending account and removes the request only when nothing of the account is left. A refused delete keeps the request with the bucket's answer and is retried; the off-host card shows how many deletions are outstanding and why. A copy the nightly run was writing while the account was deleted is taken back after the upload. The off-host card also reads the bucket's lifecycle rule and says plainly when there is none (every nightly copy kept forever) or when the credential may not read it. Every stored backup now records the key ids its content was written under, and every off-host upload records the same per key. The admin encryption view lists, per key, the stored copies that need it with the oldest date, and when the last off-host copy needing it expires. The "safe to drop" badge now requires that no backup still needs a retired key, instead of only that every row was rotated. Migration 0358 adds data_backups.inner_key_ids, the key-use table and the deletion requests.
This commit is contained in:
1 parent
405c996b6a
commit
1c7785c23b
32 files changed
+1590
-42
No files matched your search
+17
-2
@@ -7276,7 +7276,12 @@
|
||||
"neverDetail": "Für dieses Konto ist nichts angekommen. Zuletzt geprüft {when}.",
|
||||
"unknownDetail": "Noch kein nächtlicher Lauf hat dieses Konto erfasst.",
|
||||
"age": "vor {hours} h",
|
||||
"scheduleNote": "Der Upload läuft alle {hours} h. Frisch heißt innerhalb eines Laufs, fällig heißt ein Lauf blieb ohne Ergebnis, veraltet heißt zwei."
|
||||
"scheduleNote": "Der Upload läuft alle {hours} h. Frisch heißt innerhalb eines Laufs, fällig heißt ein Lauf blieb ohne Ergebnis, veraltet heißt zwei.",
|
||||
"lifecycleConfigured": "Der Bucket löscht Kopien nach {days} Tagen über seine eigene Lifecycle-Regel.",
|
||||
"lifecycleMissing": "Der Bucket hat keine Lifecycle-Regel, jede nächtliche Kopie bleibt also für immer liegen. Lege eine Regel an, die Objekte nach BACKUP_RETENTION_DAYS löscht.",
|
||||
"lifecycleUnknown": "Ob der Bucket alte Kopien löscht, ließ sich nicht lesen: Die Zugangsdaten dürfen seine Lifecycle-Regel nicht lesen. Prüfe, dass eine Regel Objekte nach BACKUP_RETENTION_DAYS löscht.",
|
||||
"pendingDeletions": "Kopien von {count} gelöschten oder geleerten Konten liegen noch im Bucket und werden entfernt.",
|
||||
"pendingDeletionsFailing": "Kopien von {count} gelöschten oder geleerten Konten liegen noch im Bucket, weil er das Löschen verweigert hat ({error}). Gib den Zugangsdaten DeleteObject; das Löschen wird jede Nacht wiederholt."
|
||||
},
|
||||
"restoreStarted": "Wiederherstellung gestartet. Der Fortschritt steht über der Liste.",
|
||||
"restoreActive": "Für dieses Konto läuft schon eine Wiederherstellung. Warte, bis sie fertig ist.",
|
||||
@@ -7364,7 +7369,17 @@
|
||||
"completed": "Abgeschlossen",
|
||||
"failed": "Fehlgeschlagen"
|
||||
},
|
||||
"coverageDetail": "Es werden nur Schlüssel-IDs und Anzahlen gezeigt — niemals Schlüsselmaterial."
|
||||
"coverageDetail": "Es werden nur Schlüssel-IDs und Anzahlen gezeigt — niemals Schlüsselmaterial.",
|
||||
"backupsTitle": "Schlüssel, die die Sicherungen noch brauchen",
|
||||
"backupsDescription": "Ein alter Schlüssel bleibt nötig, solange eine Sicherung unten ihn noch aufführt.",
|
||||
"backupsStoredLine": "Schlüssel {key}: {count} gespeicherte Kopien, die älteste vom {when}",
|
||||
"backupsUnrecordedLine": "{count} gespeicherte Kopien stammen aus der Zeit, bevor Sicherungen ihre Schlüssel festhielten (die älteste vom {when}). Sie können jeden damals vorhandenen Schlüssel brauchen: Behalte alle alten Schlüssel, bis sie ersetzt oder gelöscht sind.",
|
||||
"backupsOffhostLine": "Schlüssel {key}: Die letzte externe Kopie, die ihn braucht, kam am {when} in den Bucket; dort gebraucht bis {until}",
|
||||
"backupsOffhostLineUnknown": "Schlüssel {key}: Die letzte externe Kopie, die ihn braucht, kam am {when} in den Bucket. Wie lange der Bucket Kopien behält, ist unbekannt; behalte den Schlüssel, bis diese Kopie sicher weg ist.",
|
||||
"backupsNone": "Noch keine Sicherung hat die Schlüssel festgehalten, die sie braucht.",
|
||||
"backupsStillNeed": "Alle Zeilen sind rotiert, aber Sicherungen brauchen noch {keys}",
|
||||
"backupsUnrecordedBadge": "Alle Zeilen sind rotiert, aber ältere Sicherungen können noch einen ausgemusterten Schlüssel brauchen",
|
||||
"backupsDetail": "Die Rotation verschlüsselt jede Zeile neu, aber eine Sicherung behält den Schlüssel, unter dem ihr Inhalt geschrieben wurde, und ihre Wiederherstellung braucht ihn. Behalte einen ausgemusterten Schlüssel in ENCRYPTION_KEYS, bis er unten nicht mehr aufgeführt ist."
|
||||
}
|
||||
},
|
||||
"carrier": "Netzbetreiber",
|
||||
|
||||
+17
-2
@@ -7276,7 +7276,12 @@
|
||||
"neverDetail": "Nothing has landed for this account. Last checked {when}.",
|
||||
"unknownDetail": "No nightly run has recorded this account yet.",
|
||||
"age": "{hours} h ago",
|
||||
"scheduleNote": "The upload runs every {hours} h. Fresh means inside one run, due means one run produced nothing, stale means two did not."
|
||||
"scheduleNote": "The upload runs every {hours} h. Fresh means inside one run, due means one run produced nothing, stale means two did not.",
|
||||
"lifecycleConfigured": "The bucket deletes copies after {days} days by its own lifecycle rule.",
|
||||
"lifecycleMissing": "The bucket has no lifecycle rule, so every nightly copy is kept forever. Add a rule that expires objects after BACKUP_RETENTION_DAYS.",
|
||||
"lifecycleUnknown": "Whether the bucket deletes old copies could not be read: the credential may not read its lifecycle rule. Check that a rule expires objects after BACKUP_RETENTION_DAYS.",
|
||||
"pendingDeletions": "Copies of {count} deleted or wiped accounts are still in the bucket and are being removed.",
|
||||
"pendingDeletionsFailing": "Copies of {count} deleted or wiped accounts are still in the bucket because it refused to delete them ({error}). Give the credential DeleteObject; the deletion is retried every night."
|
||||
},
|
||||
"restoreStarted": "Restore started. Its progress shows above the list.",
|
||||
"restoreActive": "A restore of this account is already running. Wait for it to finish.",
|
||||
@@ -7364,7 +7369,17 @@
|
||||
"completed": "Completed",
|
||||
"failed": "Failed"
|
||||
},
|
||||
"coverageDetail": "Only key ids and counts are shown — never key material."
|
||||
"coverageDetail": "Only key ids and counts are shown — never key material.",
|
||||
"backupsTitle": "Keys the backups still need",
|
||||
"backupsDescription": "A retired key stays needed while any backup below still lists it.",
|
||||
"backupsStoredLine": "Key {key}: {count} stored copies, the oldest from {when}",
|
||||
"backupsUnrecordedLine": "{count} stored copies were written before backups recorded their keys (the oldest on {when}). They may need any key that existed then: keep every old key until they are replaced or deleted.",
|
||||
"backupsOffhostLine": "Key {key}: the last off-host copy that needs it went into the bucket on {when}; needed there until {until}",
|
||||
"backupsOffhostLineUnknown": "Key {key}: the last off-host copy that needs it went into the bucket on {when}. The bucket's expiry is unknown, so keep the key until you know that copy is gone.",
|
||||
"backupsNone": "No backup has recorded the keys it needs yet.",
|
||||
"backupsStillNeed": "Every row is rotated, but backups still need {keys}",
|
||||
"backupsUnrecordedBadge": "Every row is rotated, but older backups may still need a retired key",
|
||||
"backupsDetail": "Rotation re-encrypts every row, but a backup keeps the key its content was written under, and restoring it needs that key. Keep a retired key in ENCRYPTION_KEYS until nothing below lists it."
|
||||
}
|
||||
},
|
||||
"carrier": "Carrier",
|
||||
|
||||
+17
-2
@@ -7276,7 +7276,12 @@
|
||||
"neverDetail": "No ha llegado nada para esta cuenta. Última comprobación {when}.",
|
||||
"unknownDetail": "Ninguna ejecución nocturna ha registrado esta cuenta todavía.",
|
||||
"age": "hace {hours} h",
|
||||
"scheduleNote": "La subida se ejecuta cada {hours} h. Reciente significa dentro de una ejecución, pendiente que una no produjo nada, obsoleta que fueron dos."
|
||||
"scheduleNote": "La subida se ejecuta cada {hours} h. Reciente significa dentro de una ejecución, pendiente que una no produjo nada, obsoleta que fueron dos.",
|
||||
"lifecycleConfigured": "El bucket borra las copias a los {days} días con su propia regla de ciclo de vida.",
|
||||
"lifecycleMissing": "El bucket no tiene regla de ciclo de vida, así que cada copia nocturna se guarda para siempre. Añade una regla que caduque los objetos tras BACKUP_RETENTION_DAYS.",
|
||||
"lifecycleUnknown": "No se pudo leer si el bucket borra las copias antiguas: la credencial no puede leer su regla de ciclo de vida. Comprueba que una regla caduque los objetos tras BACKUP_RETENTION_DAYS.",
|
||||
"pendingDeletions": "Aún quedan en el bucket copias de {count} cuentas eliminadas o vaciadas, y se están borrando.",
|
||||
"pendingDeletionsFailing": "Aún quedan en el bucket copias de {count} cuentas eliminadas o vaciadas porque rechazó borrarlas ({error}). Da a la credencial el permiso DeleteObject; el borrado se reintenta cada noche."
|
||||
},
|
||||
"restoreStarted": "Restauración iniciada. Su progreso aparece encima de la lista.",
|
||||
"restoreActive": "Ya hay una restauración en curso para esta cuenta. Espera a que termine.",
|
||||
@@ -7364,7 +7369,17 @@
|
||||
"completed": "Completado",
|
||||
"failed": "Fallido"
|
||||
},
|
||||
"coverageDetail": "Solo se muestran los id de clave y los recuentos, nunca el material de la clave."
|
||||
"coverageDetail": "Solo se muestran los id de clave y los recuentos, nunca el material de la clave.",
|
||||
"backupsTitle": "Claves que las copias aún necesitan",
|
||||
"backupsDescription": "Una clave retirada sigue siendo necesaria mientras alguna copia de abajo la indique.",
|
||||
"backupsStoredLine": "Clave {key}: {count} copias guardadas, la más antigua del {when}",
|
||||
"backupsUnrecordedLine": "{count} copias guardadas se escribieron antes de que las copias registraran sus claves (la más antigua el {when}). Pueden necesitar cualquier clave que existiera entonces: conserva todas las claves antiguas hasta que se sustituyan o borren.",
|
||||
"backupsOffhostLine": "Clave {key}: la última copia externa que la necesita entró en el bucket el {when}; necesaria allí hasta el {until}",
|
||||
"backupsOffhostLineUnknown": "Clave {key}: la última copia externa que la necesita entró en el bucket el {when}. No se sabe cuándo caducan las copias en el bucket, así que conserva la clave hasta saber que esa copia ya no está.",
|
||||
"backupsNone": "Ninguna copia ha registrado todavía las claves que necesita.",
|
||||
"backupsStillNeed": "Todas las filas están rotadas, pero las copias aún necesitan {keys}",
|
||||
"backupsUnrecordedBadge": "Todas las filas están rotadas, pero copias antiguas pueden necesitar aún una clave retirada",
|
||||
"backupsDetail": "La rotación vuelve a cifrar cada fila, pero una copia conserva la clave con la que se escribió su contenido, y restaurarla necesita esa clave. Mantén una clave retirada en ENCRYPTION_KEYS hasta que no aparezca abajo."
|
||||
}
|
||||
},
|
||||
"carrier": "Operador",
|
||||
|
||||
+17
-2
@@ -7276,7 +7276,12 @@
|
||||
"neverDetail": "Rien n'est arrivé pour ce compte. Dernière vérification {when}.",
|
||||
"unknownDetail": "Aucun cycle nocturne n'a encore enregistré ce compte.",
|
||||
"age": "il y a {hours} h",
|
||||
"scheduleNote": "L'envoi tourne toutes les {hours} h. Récente signifie dans un cycle, attendue qu'un cycle n'a rien produit, périmée que deux n'ont rien produit."
|
||||
"scheduleNote": "L'envoi tourne toutes les {hours} h. Récente signifie dans un cycle, attendue qu'un cycle n'a rien produit, périmée que deux n'ont rien produit.",
|
||||
"lifecycleConfigured": "Le bucket supprime les copies après {days} jours grâce à sa propre règle de cycle de vie.",
|
||||
"lifecycleMissing": "Le bucket n'a aucune règle de cycle de vie : chaque copie nocturne est donc conservée indéfiniment. Ajoutez une règle qui fait expirer les objets après BACKUP_RETENTION_DAYS.",
|
||||
"lifecycleUnknown": "Impossible de savoir si le bucket supprime les anciennes copies : l'identifiant ne peut pas lire sa règle de cycle de vie. Vérifiez qu'une règle fait expirer les objets après BACKUP_RETENTION_DAYS.",
|
||||
"pendingDeletions": "Des copies de {count} comptes supprimés ou vidés sont encore dans le bucket et sont en cours de suppression.",
|
||||
"pendingDeletionsFailing": "Des copies de {count} comptes supprimés ou vidés sont encore dans le bucket, qui a refusé de les supprimer ({error}). Accordez DeleteObject à l'identifiant ; la suppression est retentée chaque nuit."
|
||||
},
|
||||
"restoreStarted": "Restauration lancée. Sa progression s'affiche au-dessus de la liste.",
|
||||
"restoreActive": "Une restauration de ce compte est déjà en cours. Attends qu'elle se termine.",
|
||||
@@ -7364,7 +7369,17 @@
|
||||
"completed": "Terminé",
|
||||
"failed": "Échoué"
|
||||
},
|
||||
"coverageDetail": "Seuls les id de clé et les décomptes sont affichés, jamais le matériel de clé."
|
||||
"coverageDetail": "Seuls les id de clé et les décomptes sont affichés, jamais le matériel de clé.",
|
||||
"backupsTitle": "Clés dont les sauvegardes ont encore besoin",
|
||||
"backupsDescription": "Une clé retirée reste nécessaire tant qu'une sauvegarde ci-dessous la mentionne.",
|
||||
"backupsStoredLine": "Clé {key} : {count} copies stockées, la plus ancienne du {when}",
|
||||
"backupsUnrecordedLine": "{count} copies stockées ont été écrites avant que les sauvegardes n'enregistrent leurs clés (la plus ancienne le {when}). Elles peuvent avoir besoin de n'importe quelle clé de l'époque : gardez toutes les anciennes clés jusqu'à leur remplacement ou suppression.",
|
||||
"backupsOffhostLine": "Clé {key} : la dernière copie externe qui en a besoin est arrivée dans le bucket le {when} ; nécessaire jusqu'au {until}",
|
||||
"backupsOffhostLineUnknown": "Clé {key} : la dernière copie externe qui en a besoin est arrivée dans le bucket le {when}. La durée de conservation du bucket est inconnue : gardez la clé jusqu'à être sûr que cette copie a disparu.",
|
||||
"backupsNone": "Aucune sauvegarde n'a encore enregistré les clés dont elle a besoin.",
|
||||
"backupsStillNeed": "Toutes les lignes sont renouvelées, mais des sauvegardes ont encore besoin de {keys}",
|
||||
"backupsUnrecordedBadge": "Toutes les lignes sont renouvelées, mais d'anciennes sauvegardes peuvent encore avoir besoin d'une clé retirée",
|
||||
"backupsDetail": "La rotation rechiffre chaque ligne, mais une sauvegarde garde la clé sous laquelle son contenu a été écrit, et sa restauration en a besoin. Gardez une clé retirée dans ENCRYPTION_KEYS tant qu'elle figure ci-dessous."
|
||||
}
|
||||
},
|
||||
"carrier": "Opérateur",
|
||||
|
||||
+17
-2
@@ -7276,7 +7276,12 @@
|
||||
"neverDetail": "Per questo account non è arrivato nulla. Ultimo controllo {when}.",
|
||||
"unknownDetail": "Nessun giro notturno ha ancora registrato questo account.",
|
||||
"age": "{hours} h fa",
|
||||
"scheduleNote": "Il caricamento gira ogni {hours} h. Recente significa entro un giro, attesa che un giro non ha prodotto nulla, obsoleta che non lo hanno fatto due."
|
||||
"scheduleNote": "Il caricamento gira ogni {hours} h. Recente significa entro un giro, attesa che un giro non ha prodotto nulla, obsoleta che non lo hanno fatto due.",
|
||||
"lifecycleConfigured": "Il bucket elimina le copie dopo {days} giorni con la propria regola di ciclo di vita.",
|
||||
"lifecycleMissing": "Il bucket non ha una regola di ciclo di vita, quindi ogni copia notturna viene conservata per sempre. Aggiungi una regola che faccia scadere gli oggetti dopo BACKUP_RETENTION_DAYS.",
|
||||
"lifecycleUnknown": "Non è stato possibile leggere se il bucket elimina le copie vecchie: la credenziale non può leggere la sua regola di ciclo di vita. Verifica che una regola faccia scadere gli oggetti dopo BACKUP_RETENTION_DAYS.",
|
||||
"pendingDeletions": "Copie di {count} account eliminati o svuotati sono ancora nel bucket e vengono rimosse.",
|
||||
"pendingDeletionsFailing": "Copie di {count} account eliminati o svuotati sono ancora nel bucket perché ha rifiutato di eliminarle ({error}). Concedi DeleteObject alla credenziale; l'eliminazione viene ritentata ogni notte."
|
||||
},
|
||||
"restoreStarted": "Ripristino avviato. L'avanzamento compare sopra l'elenco.",
|
||||
"restoreActive": "Per questo account è già in corso un ripristino. Aspetta che finisca.",
|
||||
@@ -7364,7 +7369,17 @@
|
||||
"completed": "Completato",
|
||||
"failed": "Fallito"
|
||||
},
|
||||
"coverageDetail": "Vengono mostrati solo gli id di chiave e i conteggi, mai il materiale della chiave."
|
||||
"coverageDetail": "Vengono mostrati solo gli id di chiave e i conteggi, mai il materiale della chiave.",
|
||||
"backupsTitle": "Chiavi di cui i backup hanno ancora bisogno",
|
||||
"backupsDescription": "Una chiave ritirata resta necessaria finché un backup qui sotto la elenca ancora.",
|
||||
"backupsStoredLine": "Chiave {key}: {count} copie salvate, la più vecchia del {when}",
|
||||
"backupsUnrecordedLine": "{count} copie salvate sono state scritte prima che i backup registrassero le loro chiavi (la più vecchia il {when}). Possono richiedere qualsiasi chiave esistente allora: conserva tutte le vecchie chiavi finché non vengono sostituite o eliminate.",
|
||||
"backupsOffhostLine": "Chiave {key}: l'ultima copia esterna che la richiede è entrata nel bucket il {when}; necessaria fino al {until}",
|
||||
"backupsOffhostLineUnknown": "Chiave {key}: l'ultima copia esterna che la richiede è entrata nel bucket il {when}. La scadenza del bucket è sconosciuta, quindi conserva la chiave finché non sai che quella copia non c'è più.",
|
||||
"backupsNone": "Nessun backup ha ancora registrato le chiavi di cui ha bisogno.",
|
||||
"backupsStillNeed": "Tutte le righe sono ruotate, ma i backup hanno ancora bisogno di {keys}",
|
||||
"backupsUnrecordedBadge": "Tutte le righe sono ruotate, ma backup più vecchi potrebbero ancora richiedere una chiave ritirata",
|
||||
"backupsDetail": "La rotazione cifra di nuovo ogni riga, ma un backup conserva la chiave con cui è stato scritto il suo contenuto, e ripristinarlo richiede quella chiave. Tieni una chiave ritirata in ENCRYPTION_KEYS finché compare qui sotto."
|
||||
}
|
||||
},
|
||||
"carrier": "Operatore",
|
||||
|
||||
+17
-2
@@ -7255,7 +7255,12 @@
|
||||
"neverDetail": "이 계정으로 도착한 것이 없습니다. 마지막 확인 {when}.",
|
||||
"unknownDetail": "아직 어떤 야간 실행도 이 계정을 기록하지 않았습니다.",
|
||||
"age": "{hours}시간 전",
|
||||
"scheduleNote": "업로드는 {hours}시간마다 실행됩니다. 최신은 한 번의 실행 이내, 지연은 한 번이 아무것도 만들지 못한 상태, 오래됨은 두 번이 그런 상태입니다."
|
||||
"scheduleNote": "업로드는 {hours}시간마다 실행됩니다. 최신은 한 번의 실행 이내, 지연은 한 번이 아무것도 만들지 못한 상태, 오래됨은 두 번이 그런 상태입니다.",
|
||||
"lifecycleConfigured": "버킷의 수명 주기 규칙에 따라 복사본이 {days}일 후 삭제됩니다.",
|
||||
"lifecycleMissing": "버킷에 수명 주기 규칙이 없어 매일 밤의 복사본이 영구히 보관됩니다. BACKUP_RETENTION_DAYS 후 객체를 만료시키는 규칙을 추가하세요.",
|
||||
"lifecycleUnknown": "버킷이 오래된 복사본을 삭제하는지 읽을 수 없었습니다. 자격 증명이 수명 주기 규칙을 읽지 못할 수 있습니다. BACKUP_RETENTION_DAYS 후 객체를 만료시키는 규칙이 있는지 확인하세요.",
|
||||
"pendingDeletions": "삭제되었거나 비워진 계정 {count}개의 복사본이 아직 버킷에 있으며 제거 중입니다.",
|
||||
"pendingDeletionsFailing": "버킷이 삭제를 거부하여 삭제되었거나 비워진 계정 {count}개의 복사본이 아직 남아 있습니다 ({error}). 자격 증명에 DeleteObject 권한을 부여하세요. 삭제는 매일 밤 다시 시도됩니다."
|
||||
},
|
||||
"restoreStarted": "복원을 시작했어요. 진행 상황은 목록 위에 표시돼요.",
|
||||
"restoreActive": "이 계정의 복원이 이미 진행 중이에요. 끝날 때까지 기다려 주세요.",
|
||||
@@ -7343,7 +7348,17 @@
|
||||
"completed": "완료",
|
||||
"failed": "실패"
|
||||
},
|
||||
"coverageDetail": "키 id와 개수만 보여줘요 — 키 자체는 절대 표시하지 않아요."
|
||||
"coverageDetail": "키 id와 개수만 보여줘요 — 키 자체는 절대 표시하지 않아요.",
|
||||
"backupsTitle": "백업에 아직 필요한 키",
|
||||
"backupsDescription": "아래 백업 중 하나라도 표시하는 동안 폐기한 키는 계속 필요합니다.",
|
||||
"backupsStoredLine": "키 {key}: 저장된 복사본 {count}개, 가장 오래된 것은 {when}",
|
||||
"backupsUnrecordedLine": "저장된 복사본 {count}개는 백업이 키를 기록하기 전에 작성되었습니다(가장 오래된 것은 {when}). 당시 존재하던 어떤 키든 필요할 수 있으니, 교체되거나 삭제될 때까지 이전 키를 모두 유지하세요.",
|
||||
"backupsOffhostLine": "키 {key}: 이 키가 필요한 마지막 외부 복사본이 {when}에 버킷에 저장되었습니다. {until}까지 필요합니다",
|
||||
"backupsOffhostLineUnknown": "키 {key}: 이 키가 필요한 마지막 외부 복사본이 {when}에 버킷에 저장되었습니다. 버킷의 만료 기간을 알 수 없으므로 그 복사본이 사라진 것을 확인할 때까지 키를 유지하세요.",
|
||||
"backupsNone": "아직 필요한 키를 기록한 백업이 없습니다.",
|
||||
"backupsStillNeed": "모든 행이 회전되었지만 백업에 아직 {keys}가 필요합니다",
|
||||
"backupsUnrecordedBadge": "모든 행이 회전되었지만 이전 백업에는 아직 폐기한 키가 필요할 수 있습니다",
|
||||
"backupsDetail": "회전은 모든 행을 다시 암호화하지만, 백업은 내용이 기록된 키를 그대로 유지하며 복원에는 그 키가 필요합니다. 아래에 더 이상 표시되지 않을 때까지 폐기한 키를 ENCRYPTION_KEYS에 유지하세요."
|
||||
}
|
||||
},
|
||||
"carrier": "통신사",
|
||||
|
||||
+17
-2
@@ -7276,7 +7276,12 @@
|
||||
"neverDetail": "Dla tego konta nic nie dotarło. Ostatnie sprawdzenie {when}.",
|
||||
"unknownDetail": "Żaden nocny przebieg nie zarejestrował jeszcze tego konta.",
|
||||
"age": "{hours} h temu",
|
||||
"scheduleNote": "Wysyłka działa co {hours} h. Świeża znaczy w obrębie jednego przebiegu, zaległa że jeden nic nie dał, przeterminowana że dwa."
|
||||
"scheduleNote": "Wysyłka działa co {hours} h. Świeża znaczy w obrębie jednego przebiegu, zaległa że jeden nic nie dał, przeterminowana że dwa.",
|
||||
"lifecycleConfigured": "Bucket usuwa kopie po {days} dniach zgodnie z własną regułą cyklu życia.",
|
||||
"lifecycleMissing": "Bucket nie ma reguły cyklu życia, więc każda nocna kopia zostaje na zawsze. Dodaj regułę, która usuwa obiekty po BACKUP_RETENTION_DAYS.",
|
||||
"lifecycleUnknown": "Nie udało się odczytać, czy bucket usuwa stare kopie: poświadczenia nie mogą odczytać jego reguły cyklu życia. Sprawdź, czy reguła usuwa obiekty po BACKUP_RETENTION_DAYS.",
|
||||
"pendingDeletions": "Kopie {count} usuniętych lub wyczyszczonych kont są jeszcze w buckecie i są usuwane.",
|
||||
"pendingDeletionsFailing": "Kopie {count} usuniętych lub wyczyszczonych kont są jeszcze w buckecie, bo odmówił ich usunięcia ({error}). Nadaj poświadczeniom uprawnienie DeleteObject; usuwanie jest ponawiane co noc."
|
||||
},
|
||||
"restoreStarted": "Przywracanie rozpoczęte. Postęp widać nad listą.",
|
||||
"restoreActive": "Dla tego konta trwa już przywracanie. Poczekaj, aż się skończy.",
|
||||
@@ -7364,7 +7369,17 @@
|
||||
"completed": "Zakończono",
|
||||
"failed": "Niepowodzenie"
|
||||
},
|
||||
"coverageDetail": "Pokazywane są tylko identyfikatory kluczy i liczby — nigdy materiał klucza."
|
||||
"coverageDetail": "Pokazywane są tylko identyfikatory kluczy i liczby — nigdy materiał klucza.",
|
||||
"backupsTitle": "Klucze, których kopie nadal potrzebują",
|
||||
"backupsDescription": "Wycofany klucz jest potrzebny, dopóki którakolwiek kopia poniżej go wymienia.",
|
||||
"backupsStoredLine": "Klucz {key}: {count} zapisanych kopii, najstarsza z {when}",
|
||||
"backupsUnrecordedLine": "{count} zapisanych kopii powstało, zanim kopie zaczęły zapisywać swoje klucze (najstarsza {when}). Mogą potrzebować każdego klucza, który wtedy istniał: zachowaj wszystkie stare klucze, dopóki nie zostaną zastąpione lub usunięte.",
|
||||
"backupsOffhostLine": "Klucz {key}: ostatnia kopia zewnętrzna, która go potrzebuje, trafiła do bucketu {when}; potrzebny tam do {until}",
|
||||
"backupsOffhostLineUnknown": "Klucz {key}: ostatnia kopia zewnętrzna, która go potrzebuje, trafiła do bucketu {when}. Nie wiadomo, jak długo bucket przechowuje kopie, więc zachowaj klucz, dopóki nie masz pewności, że tej kopii już nie ma.",
|
||||
"backupsNone": "Żadna kopia nie zapisała jeszcze kluczy, których potrzebuje.",
|
||||
"backupsStillNeed": "Wszystkie wiersze są zrotowane, ale kopie nadal potrzebują {keys}",
|
||||
"backupsUnrecordedBadge": "Wszystkie wiersze są zrotowane, ale starsze kopie mogą nadal potrzebować wycofanego klucza",
|
||||
"backupsDetail": "Rotacja ponownie szyfruje każdy wiersz, ale kopia zachowuje klucz, którym zapisano jej zawartość, a jej przywrócenie wymaga tego klucza. Trzymaj wycofany klucz w ENCRYPTION_KEYS, dopóki widnieje poniżej."
|
||||
}
|
||||
},
|
||||
"carrier": "Operator",
|
||||
|
||||
@@ -0,0 +1,37 @@
|
||||
-- Backups: which keys their content needs, and off-host copies that leave
|
||||
-- with the account.
|
||||
--
|
||||
-- 1. A stored backup carries encrypted columns as the stored ciphertext, and
|
||||
-- key rotation cannot reach inside a copy. Each copy now records the key
|
||||
-- ids its content was written under, so the admin encryption view can say
|
||||
-- which retired key is still needed and by how old a copy. Copies written
|
||||
-- before this release are marked unrecorded rather than assumed clean.
|
||||
ALTER TABLE "data_backups"
|
||||
ADD COLUMN "inner_key_ids" TEXT[] NOT NULL DEFAULT ARRAY[]::TEXT[],
|
||||
ADD COLUMN "inner_key_ids_recorded" BOOLEAN NOT NULL DEFAULT false;
|
||||
|
||||
-- 2. The same fact for the off-host bucket, per key rather than per object:
|
||||
-- when the nightly run last wrote an object that needed each key.
|
||||
CREATE TABLE "offhost_backup_key_use" (
|
||||
"key_id" TEXT NOT NULL,
|
||||
"first_written_at" TIMESTAMP(3) NOT NULL,
|
||||
"last_written_at" TIMESTAMP(3) NOT NULL,
|
||||
CONSTRAINT "offhost_backup_key_use_pkey" PRIMARY KEY ("key_id")
|
||||
);
|
||||
|
||||
-- 3. Accounts whose off-host objects have to be deleted: written with the
|
||||
-- account deletion or the data wipe, removed once the bucket holds nothing
|
||||
-- for the account. No foreign key, because a deleted account's row is gone
|
||||
-- by the time the job runs.
|
||||
CREATE TABLE "offhost_purge_requests" (
|
||||
"id" TEXT NOT NULL,
|
||||
"subject_id" TEXT NOT NULL,
|
||||
"reason" TEXT NOT NULL,
|
||||
"requested_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
"attempts" INTEGER NOT NULL DEFAULT 0,
|
||||
"last_attempt_at" TIMESTAMP(3),
|
||||
"last_failure" TEXT,
|
||||
CONSTRAINT "offhost_purge_requests_pkey" PRIMARY KEY ("id")
|
||||
);
|
||||
CREATE INDEX "offhost_purge_requests_requested_at_idx"
|
||||
ON "offhost_purge_requests"("requested_at");
|
||||
+54
-7
@@ -4970,19 +4970,29 @@ model PushSubscription {
|
||||
// ─── Data Backups ────────────────────────────────────────────
|
||||
|
||||
model DataBackup {
|
||||
id String @id @default(cuid())
|
||||
userId String @map("user_id")
|
||||
type String @default("WEEKLY_AUTO") // WEEKLY_AUTO, MANUAL_UPLOAD_<ms>
|
||||
id String @id @default(cuid())
|
||||
userId String @map("user_id")
|
||||
type String @default("WEEKLY_AUTO") // WEEKLY_AUTO, MANUAL_UPLOAD_<ms>
|
||||
/// The whole encrypted copy as one value: how every backup was stored up
|
||||
/// to v1.39.1. NULL for a copy stored in pieces (`chunks`), which is how
|
||||
/// every backup is written from v1.39.2 on. A row carries one or the other.
|
||||
data String?
|
||||
data String?
|
||||
/// How many pieces a chunked copy has, and the random id sealed into every
|
||||
/// one of them (`src/lib/export/backup-chunks.ts`). The reader checks both
|
||||
/// against the pieces before it releases a byte; NULL on a single-value copy.
|
||||
chunkCount Int? @map("chunk_count")
|
||||
chunkStreamId String? @map("chunk_stream_id")
|
||||
createdAt DateTime @default(now()) @map("created_at")
|
||||
chunkCount Int? @map("chunk_count")
|
||||
chunkStreamId String? @map("chunk_stream_id")
|
||||
/// The application key ids the ciphertext INSIDE this copy was written
|
||||
/// under, recorded as the copy is written (`createBackupKeyIdTextScanner`).
|
||||
/// Key rotation re-seals the copy's envelope but not what is inside it, so
|
||||
/// a key stays needed while any copy lists it. NULL for a copy written
|
||||
/// before v1.39.3, which the admin encryption view counts as unrecorded.
|
||||
/// Consumer: `GET /api/admin/encryption/status` (`backups.stored`).
|
||||
innerKeyIds String[] @default([]) @map("inner_key_ids")
|
||||
/// Whether `innerKeyIds` was recorded; false on every copy written before
|
||||
/// v1.39.3. Consumer: the same status read (`backups.unrecorded`).
|
||||
innerKeyIdsRecorded Boolean @default(false) @map("inner_key_ids_recorded")
|
||||
createdAt DateTime @default(now()) @map("created_at")
|
||||
|
||||
user User @relation(fields: [userId], references: [id], onDelete: Cascade)
|
||||
chunks DataBackupChunk[]
|
||||
@@ -5148,6 +5158,43 @@ model OffhostBackupState {
|
||||
@@map("offhost_backup_state")
|
||||
}
|
||||
|
||||
/// Per application key id, when the nightly off-host run last put an object
|
||||
/// in the bucket whose inner ciphertext needed that key. The bucket's
|
||||
/// lifecycle rule keeps each object for its retention, so a retired key is
|
||||
/// needed until that long after this date. Not per account: the question it
|
||||
/// answers is about keys. Written by `runOffhostBackup`; consumer:
|
||||
/// `GET /api/admin/encryption/status` (`backups.offhost`).
|
||||
model OffhostBackupKeyUse {
|
||||
keyId String @id @map("key_id")
|
||||
firstWrittenAt DateTime @map("first_written_at")
|
||||
lastWrittenAt DateTime @map("last_written_at")
|
||||
|
||||
@@map("offhost_backup_key_use")
|
||||
}
|
||||
|
||||
/// An account whose off-host copies have to leave the bucket: it was deleted,
|
||||
/// or its data was wiped. Written in the same transaction as the deletion, so
|
||||
/// the request cannot be lost between the two, and deleted once every object
|
||||
/// for the account is gone. `subjectId` is deliberately not a relation: for a
|
||||
/// deleted account the user row is already gone. Written by
|
||||
/// `requestOffhostPurge`; consumer: `processOffhostPurges` (the
|
||||
/// `offhost-backup-purge` job) and the off-host card's pending count behind
|
||||
/// `GET /api/admin/backups`.
|
||||
model OffhostPurgeRequest {
|
||||
id String @id @default(cuid())
|
||||
subjectId String @map("subject_id")
|
||||
/// `account_deleted`, `data_wiped`, `managed_profile_deleted` or `admin_wipe`.
|
||||
reason String
|
||||
requestedAt DateTime @default(now()) @map("requested_at")
|
||||
attempts Int @default(0)
|
||||
lastAttemptAt DateTime? @map("last_attempt_at")
|
||||
/// The bucket's last refusal, trimmed; shown to the operator.
|
||||
lastFailure String? @map("last_failure")
|
||||
|
||||
@@index([requestedAt])
|
||||
@@map("offhost_purge_requests")
|
||||
}
|
||||
|
||||
// ─── Apple Health export.zip import (v1.4.34) ───────────────
|
||||
//
|
||||
// Mirrors a `pg-boss` `apple-health-import` job's lifecycle so the
|
||||
|
||||
@@ -29,12 +29,23 @@ vi.mock("@/lib/db", () => ({
|
||||
prisma: {
|
||||
$queryRaw: (...a: unknown[]) => queryRawMock(...a),
|
||||
user: { findMany: (...a: unknown[]) => findManyMock(...a) },
|
||||
offhostPurgeRequest: {
|
||||
count: async () => 2,
|
||||
findFirst: async () => ({
|
||||
requestedAt: new Date("2026-09-20T10:00:00.000Z"),
|
||||
lastFailure: "AccessDenied",
|
||||
}),
|
||||
},
|
||||
},
|
||||
}));
|
||||
|
||||
const configuredMock = vi.fn(() => true);
|
||||
vi.mock("@/lib/jobs/offhost-backup", () => ({
|
||||
offhostBackupConfigured: () => configuredMock(),
|
||||
probeOffhostLifecycle: async () => ({
|
||||
state: "missing",
|
||||
expirationDays: null,
|
||||
}),
|
||||
}));
|
||||
|
||||
import { GET } from "../route";
|
||||
@@ -139,10 +150,12 @@ describe("GET /api/admin/backups — off-host freshness", () => {
|
||||
|
||||
const body = await read();
|
||||
|
||||
expect(body.offhost).toEqual({
|
||||
expect(body.offhost).toMatchObject({
|
||||
configured: false,
|
||||
periodHours: 24,
|
||||
rows: [],
|
||||
// No bucket to ask on a host that is not configured.
|
||||
lifecycle: { state: "unknown", expirationDays: null },
|
||||
});
|
||||
});
|
||||
|
||||
@@ -196,4 +209,20 @@ describe("GET /api/admin/backups — off-host freshness", () => {
|
||||
]);
|
||||
expect(JSON.stringify(body.offhost)).not.toMatch(/bucket|endpoint|secret/i);
|
||||
});
|
||||
|
||||
it("carries the bucket's lifecycle verdict and the deletions it still owes", async () => {
|
||||
configuredMock.mockReturnValue(true);
|
||||
queryRawMock.mockResolvedValue([]);
|
||||
findManyMock.mockResolvedValue([]);
|
||||
const body = await read();
|
||||
expect(body.offhost.lifecycle).toEqual({
|
||||
state: "missing",
|
||||
expirationDays: null,
|
||||
});
|
||||
expect(body.offhost.pendingDeletions).toEqual({
|
||||
count: 2,
|
||||
oldestRequestedAt: "2026-09-20T10:00:00.000Z",
|
||||
lastFailure: "AccessDenied",
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -27,7 +27,10 @@ import { annotate } from "@/lib/logging/context";
|
||||
import { DATA_BACKUP_QUEUE } from "@/lib/jobs/data-backup-policy";
|
||||
import { readLastQueueRun } from "@/lib/jobs/job-failures";
|
||||
import { summariseBackupSchedule } from "@/lib/jobs/backup-schedule-status";
|
||||
import { offhostBackupConfigured } from "@/lib/jobs/offhost-backup";
|
||||
import {
|
||||
offhostBackupConfigured,
|
||||
probeOffhostLifecycle,
|
||||
} from "@/lib/jobs/offhost-backup";
|
||||
import {
|
||||
classifyOffhostBackup,
|
||||
OFFHOST_BACKUP_PERIOD_HOURS,
|
||||
@@ -164,6 +167,17 @@ export const GET = apiHandler(async () => {
|
||||
a.username.localeCompare(b.username),
|
||||
);
|
||||
|
||||
const [purgeCount, oldestPurge, lifecycle] = await Promise.all([
|
||||
prisma.offhostPurgeRequest.count(),
|
||||
prisma.offhostPurgeRequest.findFirst({
|
||||
orderBy: { requestedAt: "asc" },
|
||||
select: { requestedAt: true, lastFailure: true },
|
||||
}),
|
||||
configured
|
||||
? probeOffhostLifecycle()
|
||||
: Promise.resolve({ state: "unknown" as const, expirationDays: null }),
|
||||
]);
|
||||
|
||||
const payload: BackupsList = {
|
||||
rows: list,
|
||||
// Matches the retention window the backup-prune job enforces so the
|
||||
@@ -182,6 +196,12 @@ export const GET = apiHandler(async () => {
|
||||
// An unconfigured host has nothing to list, and the card says so rather
|
||||
// than painting an empty table that reads as "no problems".
|
||||
rows: configured ? offhostRows : [],
|
||||
pendingDeletions: {
|
||||
count: purgeCount,
|
||||
oldestRequestedAt: oldestPurge?.requestedAt.toISOString() ?? null,
|
||||
lastFailure: oldestPurge?.lastFailure ?? null,
|
||||
},
|
||||
lifecycle,
|
||||
},
|
||||
};
|
||||
|
||||
|
||||
@@ -1,3 +1,8 @@
|
||||
import {
|
||||
kickOffhostPurge,
|
||||
PURGE_ALL_SUBJECTS,
|
||||
requestOffhostPurge,
|
||||
} from "@/lib/jobs/offhost-purge";
|
||||
import { prisma } from "@/lib/db";
|
||||
import { apiHandler, requireAdmin } from "@/lib/api-handler";
|
||||
import { auditLog } from "@/lib/auth/audit";
|
||||
@@ -108,6 +113,9 @@ export const DELETE = apiHandler(async (request: NextRequest) => {
|
||||
// columns does not. Same classification contract as the model list.
|
||||
await tx.user.updateMany({ data: USER_RESET });
|
||||
|
||||
// Every account's nightly off-host copies go too.
|
||||
await requestOffhostPurge(tx, PURGE_ALL_SUBJECTS, "admin_wipe");
|
||||
|
||||
return perModel;
|
||||
},
|
||||
{
|
||||
@@ -138,6 +146,7 @@ export const DELETE = apiHandler(async (request: NextRequest) => {
|
||||
// v1.16.9 — the wipe touched every user's rows; clear every cache
|
||||
// bucket so no per-user payload survives the reset.
|
||||
invalidateAllCaches();
|
||||
await kickOffhostPurge();
|
||||
|
||||
return apiSuccess({ cleared: true, deletedRows, models: counts });
|
||||
});
|
||||
@@ -19,6 +19,10 @@ import { apiHandler, requireAdmin } from "@/lib/api-handler";
|
||||
import { apiSuccess } from "@/lib/api-response";
|
||||
import { annotate } from "@/lib/logging/context";
|
||||
import { getConfiguredKeyIds } from "@/lib/crypto";
|
||||
import {
|
||||
offhostBackupConfigured,
|
||||
probeOffhostLifecycle,
|
||||
} from "@/lib/jobs/offhost-backup";
|
||||
import {
|
||||
scanCorpus,
|
||||
type CorpusClient,
|
||||
@@ -100,6 +104,100 @@ async function readRotationState(): Promise<RotationState> {
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Which keys the backups still need.
|
||||
*
|
||||
* Rotation re-encrypts every row in the database and re-seals every stored
|
||||
* backup's envelope, but the content INSIDE a backup keeps the key it was
|
||||
* written under: a restore writes it back verbatim. So "every row is on the
|
||||
* active key" is not the same as "the old key can go". This answers the second
|
||||
* question per key id, from what each copy recorded as it was written: the
|
||||
* stored copies that need it and the oldest of them, and for the off-host
|
||||
* bucket the last night an object needing it went in, plus how long the
|
||||
* bucket's lifecycle rule keeps objects after that.
|
||||
*/
|
||||
export interface BackupKeyNeeds {
|
||||
stored: Array<{ keyId: string; copies: number; oldestAt: string }>;
|
||||
/** Copies written before v1.39.3, which recorded no key ids. */
|
||||
unrecorded: { copies: number; oldestAt: string | null };
|
||||
offhost: Array<{
|
||||
keyId: string;
|
||||
firstWrittenAt: string;
|
||||
lastWrittenAt: string;
|
||||
/** When the last object needing this key expires; null when unknown. */
|
||||
neededUntil: string | null;
|
||||
}>;
|
||||
/** The bucket's expiry, when it has one the credential may read. */
|
||||
offhostExpirationDays: number | null;
|
||||
/** Retired key ids some backup still needs (never the active one). */
|
||||
retiredKeysStillNeeded: string[];
|
||||
}
|
||||
|
||||
async function readBackupKeyNeeds(
|
||||
activeKeyId: string,
|
||||
now: Date,
|
||||
): Promise<BackupKeyNeeds> {
|
||||
const [stored, unrecorded, offhostRows] = await Promise.all([
|
||||
prisma.$queryRaw<
|
||||
Array<{ key_id: string; copies: number; oldest_at: Date }>
|
||||
>`
|
||||
SELECT k AS key_id, count(*)::int AS copies, min(created_at) AS oldest_at
|
||||
FROM data_backups, unnest(inner_key_ids) AS k
|
||||
WHERE inner_key_ids_recorded
|
||||
GROUP BY k
|
||||
ORDER BY k
|
||||
`,
|
||||
prisma.dataBackup.aggregate({
|
||||
where: { innerKeyIdsRecorded: false },
|
||||
_count: { _all: true },
|
||||
_min: { createdAt: true },
|
||||
}),
|
||||
prisma.offhostBackupKeyUse.findMany({ orderBy: { keyId: "asc" } }),
|
||||
]);
|
||||
const lifecycle =
|
||||
offhostRows.length > 0 && offhostBackupConfigured()
|
||||
? await probeOffhostLifecycle()
|
||||
: null;
|
||||
const expirationDays =
|
||||
lifecycle?.state === "configured" ? lifecycle.expirationDays : null;
|
||||
|
||||
const offhost = offhostRows.map((row) => ({
|
||||
keyId: row.keyId,
|
||||
firstWrittenAt: row.firstWrittenAt.toISOString(),
|
||||
lastWrittenAt: row.lastWrittenAt.toISOString(),
|
||||
neededUntil:
|
||||
expirationDays === null
|
||||
? null
|
||||
: new Date(
|
||||
row.lastWrittenAt.getTime() + (expirationDays + 1) * 86_400_000,
|
||||
).toISOString(),
|
||||
}));
|
||||
|
||||
const needed = new Set<string>();
|
||||
for (const row of stored) needed.add(row.key_id);
|
||||
for (const row of offhost) {
|
||||
if (row.neededUntil === null || Date.parse(row.neededUntil) > +now) {
|
||||
needed.add(row.keyId);
|
||||
}
|
||||
}
|
||||
needed.delete(activeKeyId);
|
||||
|
||||
return {
|
||||
stored: stored.map((row) => ({
|
||||
keyId: row.key_id,
|
||||
copies: row.copies,
|
||||
oldestAt: row.oldest_at.toISOString(),
|
||||
})),
|
||||
unrecorded: {
|
||||
copies: unrecorded._count._all,
|
||||
oldestAt: unrecorded._min.createdAt?.toISOString() ?? null,
|
||||
},
|
||||
offhost,
|
||||
offhostExpirationDays: expirationDays,
|
||||
retiredKeysStillNeeded: [...needed].sort(),
|
||||
};
|
||||
}
|
||||
|
||||
export const GET = apiHandler(async () => {
|
||||
await requireAdmin();
|
||||
annotate({ action: { name: "admin.encryption.status" } });
|
||||
@@ -114,6 +212,7 @@ export const GET = apiHandler(async () => {
|
||||
}
|
||||
|
||||
const rotation = await readRotationState();
|
||||
const backups = await readBackupKeyNeeds(scan.activeKeyId, new Date());
|
||||
|
||||
annotate({
|
||||
meta: {
|
||||
@@ -133,5 +232,14 @@ export const GET = apiHandler(async () => {
|
||||
staleRows: scan.staleRows,
|
||||
columns: scan.columns,
|
||||
rotation,
|
||||
backups,
|
||||
// Every row rotated AND no backup still needs a retired key: the one
|
||||
// state in which dropping an old key loses nothing. Copies that recorded
|
||||
// no key ids count against it, because they may hold any key that
|
||||
// existed when they were written.
|
||||
safeToDropRetiredKeys:
|
||||
scan.rotationComplete &&
|
||||
backups.retiredKeysStillNeeded.length === 0 &&
|
||||
backups.unrecorded.copies === 0,
|
||||
});
|
||||
});
|
||||
@@ -8,6 +8,10 @@ import {
|
||||
MFA_STEP_UP_MAX_AGE_SECONDS,
|
||||
} from "@/lib/api-handler";
|
||||
import { annotate } from "@/lib/logging/context";
|
||||
import {
|
||||
kickOffhostPurge,
|
||||
requestOffhostPurge,
|
||||
} from "@/lib/jobs/offhost-purge";
|
||||
import {
|
||||
deleteGuardianAccountWithLifecycle,
|
||||
LastManagedGuardianError,
|
||||
@@ -101,6 +105,10 @@ export const DELETE = apiHandler(async (request: NextRequest) => {
|
||||
// genuinely complete erasure.
|
||||
await tx.auditLog.deleteMany({ where: { userId } });
|
||||
|
||||
// The nightly off-host copies go with the account. The request commits
|
||||
// with the deletion; the purge job deletes the objects.
|
||||
await requestOffhostPurge(tx, userId, "account_deleted");
|
||||
|
||||
// Delete user — all other related data is removed via onDelete: Cascade.
|
||||
await tx.user.delete({ where: { id: userId } });
|
||||
});
|
||||
@@ -115,6 +123,8 @@ export const DELETE = apiHandler(async (request: NextRequest) => {
|
||||
throw error;
|
||||
}
|
||||
|
||||
await kickOffhostPurge();
|
||||
|
||||
annotate({
|
||||
action: { name: "settings.account.delete" },
|
||||
meta: { username },
|
||||
|
||||
@@ -1,5 +1,9 @@
|
||||
import { prisma } from "@/lib/db";
|
||||
import { auditLog } from "@/lib/auth/audit";
|
||||
import {
|
||||
kickOffhostPurge,
|
||||
requestOffhostPurge,
|
||||
} from "@/lib/jobs/offhost-purge";
|
||||
import { apiSuccess, apiError, getClientIp } from "@/lib/api-response";
|
||||
import { NextRequest } from "next/server";
|
||||
import {
|
||||
@@ -101,6 +105,11 @@ export const DELETE = apiHandler(async (request: NextRequest) => {
|
||||
// does not. Same classification contract as the model list.
|
||||
await tx.user.update({ where: { id: userId }, data: USER_RESET });
|
||||
|
||||
// The nightly off-host copies of the record are part of "all your
|
||||
// data". The request commits with the wipe; the purge job deletes
|
||||
// the objects and removes it.
|
||||
await requestOffhostPurge(tx, userId, "data_wiped");
|
||||
|
||||
return perModel;
|
||||
},
|
||||
{
|
||||
@@ -144,6 +153,7 @@ export const DELETE = apiHandler(async (request: NextRequest) => {
|
||||
// v1.16.9 — every cached payload was built on the rows just deleted;
|
||||
// hard-evict the user's buckets so no surface serves pre-wipe data.
|
||||
invalidateUserData(userId);
|
||||
await kickOffhostPurge();
|
||||
|
||||
return apiSuccess({ cleared: true, deletedRows, models: counts });
|
||||
});
|
||||
@@ -23,6 +23,11 @@ vi.mock("@tanstack/react-query", () => ({
|
||||
import { I18nProvider } from "@/lib/i18n/context";
|
||||
import { OffhostBackupsSection } from "../offhost-backups-section";
|
||||
|
||||
const QUIET = {
|
||||
pendingDeletions: { count: 0, oldestRequestedAt: null, lastFailure: null },
|
||||
lifecycle: { state: "configured" as const, expirationDays: 30 },
|
||||
};
|
||||
|
||||
function render(data: BackupsList["offhost"] | null): string {
|
||||
queryResult.current = {
|
||||
data:
|
||||
@@ -46,7 +51,12 @@ function render(data: BackupsList["offhost"] | null): string {
|
||||
|
||||
describe("<OffhostBackupsSection>", () => {
|
||||
it("says off-host backup is not configured rather than showing an empty list", () => {
|
||||
const html = render({ configured: false, periodHours: 24, rows: [] });
|
||||
const html = render({
|
||||
configured: false,
|
||||
periodHours: 24,
|
||||
rows: [],
|
||||
...QUIET,
|
||||
});
|
||||
expect(html).toContain("Off-host backup is not configured");
|
||||
expect(html).not.toContain('data-slot="offhost-backup-rows"');
|
||||
});
|
||||
@@ -55,6 +65,7 @@ describe("<OffhostBackupsSection>", () => {
|
||||
const html = render({
|
||||
configured: true,
|
||||
periodHours: 24,
|
||||
...QUIET,
|
||||
rows: [
|
||||
{
|
||||
userId: "u1",
|
||||
@@ -109,4 +120,33 @@ describe("<OffhostBackupsSection>", () => {
|
||||
// Never the bucket's own coordinates.
|
||||
expect(html).not.toMatch(/secret|access[- ]key/i);
|
||||
});
|
||||
|
||||
it("says when the bucket keeps copies forever, and when deletions are stuck", () => {
|
||||
const html = render({
|
||||
configured: true,
|
||||
periodHours: 24,
|
||||
rows: [],
|
||||
pendingDeletions: {
|
||||
count: 2,
|
||||
oldestRequestedAt: "2026-09-20T10:00:00.000Z",
|
||||
lastFailure: "AccessDenied",
|
||||
},
|
||||
lifecycle: { state: "missing", expirationDays: null },
|
||||
});
|
||||
expect(html).toContain('data-lifecycle-state="missing"');
|
||||
expect(html).toContain("no lifecycle rule");
|
||||
expect(html).toContain('data-slot="offhost-pending-deletions"');
|
||||
expect(html).toContain("AccessDenied");
|
||||
});
|
||||
|
||||
it("states the expiry the bucket enforces", () => {
|
||||
const html = render({
|
||||
configured: true,
|
||||
periodHours: 24,
|
||||
rows: [],
|
||||
...QUIET,
|
||||
});
|
||||
expect(html).toContain("after 30 days");
|
||||
expect(html).not.toContain('data-slot="offhost-pending-deletions"');
|
||||
});
|
||||
});
|
||||
@@ -13,7 +13,13 @@
|
||||
*/
|
||||
|
||||
import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query";
|
||||
import { KeyRound, Loader2, RotateCw, ShieldCheck } from "lucide-react";
|
||||
import {
|
||||
Archive,
|
||||
KeyRound,
|
||||
Loader2,
|
||||
RotateCw,
|
||||
ShieldCheck,
|
||||
} from "lucide-react";
|
||||
import { toast } from "sonner";
|
||||
import {
|
||||
AlertDialog,
|
||||
@@ -32,7 +38,7 @@ import { StatTile } from "@/components/admin/_stat-tile";
|
||||
import { Button } from "@/components/ui/button";
|
||||
import { SettingsCard } from "@/components/settings/settings-card";
|
||||
import { SettingsCardHeader } from "@/components/settings/_card-header";
|
||||
import { useTranslations } from "@/lib/i18n/context";
|
||||
import { useFormatters, useTranslations } from "@/lib/i18n/context";
|
||||
import { queryKeys } from "@/lib/query-keys";
|
||||
import { apiFetchRaw, apiGet } from "@/lib/api/api-fetch";
|
||||
import { getApiErrorMessage } from "./_shared";
|
||||
@@ -60,10 +66,25 @@ interface EncryptionStatus {
|
||||
lastCompletedAt: string | null;
|
||||
lastResult: { scanned: number; rotated: number; errors: number } | null;
|
||||
};
|
||||
/** Which keys the backups still need; see the status route. */
|
||||
backups: {
|
||||
stored: Array<{ keyId: string; copies: number; oldestAt: string }>;
|
||||
unrecorded: { copies: number; oldestAt: string | null };
|
||||
offhost: Array<{
|
||||
keyId: string;
|
||||
firstWrittenAt: string;
|
||||
lastWrittenAt: string;
|
||||
neededUntil: string | null;
|
||||
}>;
|
||||
offhostExpirationDays: number | null;
|
||||
retiredKeysStillNeeded: string[];
|
||||
};
|
||||
safeToDropRetiredKeys: boolean;
|
||||
}
|
||||
|
||||
export function EncryptionSection() {
|
||||
const { t } = useTranslations();
|
||||
const fmt = useFormatters();
|
||||
const queryClient = useQueryClient();
|
||||
|
||||
const statusQuery = useQuery({
|
||||
@@ -166,10 +187,20 @@ export function EncryptionSection() {
|
||||
value={s.staleRows.toLocaleString()}
|
||||
/>
|
||||
</div>
|
||||
{s.rotationComplete ? (
|
||||
{s.safeToDropRetiredKeys ? (
|
||||
<Badge className="border-success/40 bg-success/15 text-success">
|
||||
{t("admin.section.encryption.safeToDropLegacy")}
|
||||
</Badge>
|
||||
) : s.rotationComplete ? (
|
||||
// Every row is on the active key, and that is not the whole
|
||||
// answer: a backup keeps the key its content was written under.
|
||||
<Badge variant="secondary" data-slot="encryption-backups-need-keys">
|
||||
{s.backups.retiredKeysStillNeeded.length > 0
|
||||
? t("admin.section.encryption.backupsStillNeed", {
|
||||
keys: s.backups.retiredKeysStillNeeded.join(", "),
|
||||
})
|
||||
: t("admin.section.encryption.backupsUnrecordedBadge")}
|
||||
</Badge>
|
||||
) : (
|
||||
<Badge variant="secondary">
|
||||
{t("admin.section.encryption.rotationIncomplete", {
|
||||
@@ -179,6 +210,56 @@ export function EncryptionSection() {
|
||||
)}
|
||||
</SettingsCard>
|
||||
|
||||
{/* ── Keys the backups still need ───────────────────────────── */}
|
||||
<SettingsCard>
|
||||
<SettingsCardHeader
|
||||
icon={Archive}
|
||||
title={t("admin.section.encryption.backupsTitle")}
|
||||
description={t("admin.section.encryption.backupsDescription")}
|
||||
/>
|
||||
<p className="text-sm">{t("admin.section.encryption.backupsDetail")}</p>
|
||||
<ul className="space-y-1 text-sm" data-slot="encryption-backup-keys">
|
||||
{s.backups.stored.map((row) => (
|
||||
<li key={`stored-${row.keyId}`}>
|
||||
{t("admin.section.encryption.backupsStoredLine", {
|
||||
key: row.keyId,
|
||||
count: row.copies,
|
||||
when: fmt.dateTime(row.oldestAt),
|
||||
})}
|
||||
</li>
|
||||
))}
|
||||
{s.backups.unrecorded.copies > 0 && s.backups.unrecorded.oldestAt ? (
|
||||
<li>
|
||||
{t("admin.section.encryption.backupsUnrecordedLine", {
|
||||
count: s.backups.unrecorded.copies,
|
||||
when: fmt.dateTime(s.backups.unrecorded.oldestAt),
|
||||
})}
|
||||
</li>
|
||||
) : null}
|
||||
{s.backups.offhost.map((row) => (
|
||||
<li key={`offhost-${row.keyId}`}>
|
||||
{row.neededUntil
|
||||
? t("admin.section.encryption.backupsOffhostLine", {
|
||||
key: row.keyId,
|
||||
when: fmt.dateTime(row.lastWrittenAt),
|
||||
until: fmt.dateTime(row.neededUntil),
|
||||
})
|
||||
: t("admin.section.encryption.backupsOffhostLineUnknown", {
|
||||
key: row.keyId,
|
||||
when: fmt.dateTime(row.lastWrittenAt),
|
||||
})}
|
||||
</li>
|
||||
))}
|
||||
{s.backups.stored.length === 0 &&
|
||||
s.backups.unrecorded.copies === 0 &&
|
||||
s.backups.offhost.length === 0 ? (
|
||||
<li className="text-muted-foreground">
|
||||
{t("admin.section.encryption.backupsNone")}
|
||||
</li>
|
||||
) : null}
|
||||
</ul>
|
||||
</SettingsCard>
|
||||
|
||||
{/* ── Rotation status + trigger ──────────────────────────────── */}
|
||||
<SettingsCard>
|
||||
<SettingsCardHeader
|
||||
|
||||
@@ -11,11 +11,12 @@
|
||||
* been refusing the worker's signature since March.
|
||||
*
|
||||
* Every verdict is computed server-side from the ledger the worker writes. The
|
||||
* page never lists the bucket: the worker's grant is deliberately PutObject,
|
||||
* GetObject and AbortMultipartUpload — the three the runbook documents, the
|
||||
* abort being what the multipart path needs — and putting a credentialed
|
||||
* listing call on the render path of an admin page would be a poor trade for
|
||||
* a number the database already holds.
|
||||
* page never lists the bucket: putting a credentialed listing call on the
|
||||
* render path of an admin page would be a poor trade for a number the database
|
||||
* already holds. It does read the bucket's lifecycle rule (one small call,
|
||||
* cached for ten minutes on the server), because whether old copies expire at
|
||||
* all is something only the bucket knows, and it states deletions of deleted
|
||||
* or wiped accounts that the bucket has not carried out yet.
|
||||
*/
|
||||
|
||||
import { useQuery } from "@tanstack/react-query";
|
||||
@@ -191,6 +192,35 @@ export function OffhostBackupsSection() {
|
||||
hours: offhost.periodHours,
|
||||
})}
|
||||
</p>
|
||||
<p
|
||||
className="text-xs"
|
||||
data-slot="offhost-lifecycle"
|
||||
data-lifecycle-state={offhost.lifecycle.state}
|
||||
>
|
||||
{offhost.lifecycle.state === "configured"
|
||||
? t("admin.section.backups.offhost.lifecycleConfigured", {
|
||||
days: offhost.lifecycle.expirationDays ?? 0,
|
||||
})
|
||||
: offhost.lifecycle.state === "missing"
|
||||
? t("admin.section.backups.offhost.lifecycleMissing")
|
||||
: t("admin.section.backups.offhost.lifecycleUnknown")}
|
||||
</p>
|
||||
{offhost.pendingDeletions.count > 0 ? (
|
||||
<p
|
||||
role="status"
|
||||
className="text-xs"
|
||||
data-slot="offhost-pending-deletions"
|
||||
>
|
||||
{offhost.pendingDeletions.lastFailure
|
||||
? t("admin.section.backups.offhost.pendingDeletionsFailing", {
|
||||
count: offhost.pendingDeletions.count,
|
||||
error: offhost.pendingDeletions.lastFailure,
|
||||
})
|
||||
: t("admin.section.backups.offhost.pendingDeletions", {
|
||||
count: offhost.pendingDeletions.count,
|
||||
})}
|
||||
</p>
|
||||
) : null}
|
||||
</div>
|
||||
) : null}
|
||||
</SettingsCard>
|
||||
|
||||
@@ -293,6 +293,10 @@ export const INSTANCE_SCOPED: Readonly<Record<string, string>> = {
|
||||
AppSettings: "instance-wide configuration singleton",
|
||||
RateLimit: "instance-wide rate-limit buckets, keyed by string, self-expiring",
|
||||
HostMetric: "instance-wide host telemetry, not attributable to an account",
|
||||
OffhostBackupKeyUse:
|
||||
"per encryption key, when the nightly off-host run last wrote an object needing it; says nothing about any account",
|
||||
OffhostPurgeRequest:
|
||||
"the deletions an account deletion or wipe asked of the off-host bucket; written by the wipe itself and removed once the bucket holds nothing of the account",
|
||||
CycleSymptomCategory:
|
||||
"seeded catalogue shared by every account; a user's custom symptoms live on CycleSymptom",
|
||||
IllnessSymptom: "seeded symptom catalogue shared by every account",
|
||||
|
||||
@@ -34,6 +34,7 @@ import {
|
||||
type BackupJsonProducer,
|
||||
type PackBackupOptions,
|
||||
} from "@/lib/export/backup-blob";
|
||||
import { createBackupKeyIdTextScanner } from "@/lib/export/backup-key-ids";
|
||||
import {
|
||||
BackupIntegrityError,
|
||||
newChunkStreamId,
|
||||
@@ -112,6 +113,11 @@ export async function storeBackupBlob(
|
||||
await tx.dataBackupChunk.deleteMany({ where: { backupId: row.id } });
|
||||
|
||||
const streamId = newChunkStreamId();
|
||||
// The key ids the copy's content was written under, read from the JSON
|
||||
// on its way into the pieces. Rotation re-seals the pieces but never
|
||||
// what is inside them, so this is what tells an operator how long a
|
||||
// retired key is still needed (`GET /api/admin/encryption/status`).
|
||||
const keyScanner = createBackupKeyIdTextScanner();
|
||||
const { chunks, bytes } = await packBackupChunks(
|
||||
async (sealed, seq) => {
|
||||
await tx.dataBackupChunk.create({
|
||||
@@ -120,7 +126,11 @@ export async function storeBackupBlob(
|
||||
});
|
||||
},
|
||||
streamId,
|
||||
producer,
|
||||
(write) =>
|
||||
producer(async (chunk) => {
|
||||
keyScanner.feed(chunk);
|
||||
await write(chunk);
|
||||
}),
|
||||
options,
|
||||
);
|
||||
|
||||
@@ -135,6 +145,8 @@ export async function storeBackupBlob(
|
||||
...(input.ownerAfterRead ? { userId: input.ownerAfterRead() } : {}),
|
||||
chunkCount: chunks,
|
||||
chunkStreamId: streamId,
|
||||
innerKeyIds: keyScanner.keyIds(),
|
||||
innerKeyIdsRecorded: true,
|
||||
createdAt: new Date(),
|
||||
},
|
||||
});
|
||||
|
||||
@@ -32,6 +32,14 @@ import {
|
||||
const ENC_KEY =
|
||||
"0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef";
|
||||
|
||||
/** The purge ledger and key-use rows the run touches after each upload. */
|
||||
function purgeLedgerMocks() {
|
||||
return {
|
||||
offhostPurgeRequest: { count: vi.fn().mockResolvedValue(0) },
|
||||
offhostBackupKeyUse: { upsert: vi.fn() },
|
||||
};
|
||||
}
|
||||
|
||||
describe("offhost-backup envelope", () => {
|
||||
it("encrypts and decrypts JSON round-trip", () => {
|
||||
const key = Buffer.from(ENC_KEY, "hex");
|
||||
@@ -184,8 +192,10 @@ describe("runOffhostBackup", () => {
|
||||
s3.store.set("2020-01-01/user-old.json.enc", Buffer.from([0]));
|
||||
|
||||
const prisma = {
|
||||
...purgeLedgerMocks(),
|
||||
offhostBackupState: { upsert: vi.fn() },
|
||||
user: {
|
||||
count: vi.fn().mockResolvedValue(1),
|
||||
findMany: vi.fn().mockResolvedValue([{ id: "u1" }, { id: "u2" }]),
|
||||
},
|
||||
measurement: { findMany: vi.fn().mockResolvedValue([]) },
|
||||
@@ -248,11 +258,43 @@ describe("runOffhostBackup", () => {
|
||||
expect(first.create.lastSuccessAt).toBeInstanceOf(Date);
|
||||
});
|
||||
|
||||
it("takes back the copy of an account deleted while it was written", async () => {
|
||||
const s3 = makeS3Mock();
|
||||
const ledger = purgeLedgerMocks();
|
||||
const prisma = {
|
||||
...ledger,
|
||||
offhostBackupState: { upsert: vi.fn() },
|
||||
user: {
|
||||
// u1 is gone by the time its upload lands; u2 is not.
|
||||
count: vi.fn(async ({ where }: { where: { id: string } }) =>
|
||||
where.id === "u1" ? 0 : 1,
|
||||
),
|
||||
findMany: vi.fn().mockResolvedValue([{ id: "u1" }, { id: "u2" }]),
|
||||
},
|
||||
};
|
||||
const report = await runOffhostBackup(
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
prisma as any,
|
||||
s3,
|
||||
new Date("2026-05-08T03:00:00Z"),
|
||||
);
|
||||
expect(s3.store.has("2026-05-08/user-u1.json.enc")).toBe(false);
|
||||
expect(s3.store.has("2026-05-08/user-u2.json.enc")).toBe(true);
|
||||
expect(s3.deleteObject).toHaveBeenCalledWith("2026-05-08/user-u1.json.enc");
|
||||
expect(report.uploaded).toBe(1);
|
||||
// No ledger row for an account that no longer exists.
|
||||
expect(prisma.offhostBackupState.upsert).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("uploads the canonical builder output without reshaping it", async () => {
|
||||
const s3 = makeS3Mock();
|
||||
const prisma = {
|
||||
...purgeLedgerMocks(),
|
||||
offhostBackupState: { upsert: vi.fn() },
|
||||
user: { findMany: vi.fn().mockResolvedValue([{ id: "u1" }]) },
|
||||
user: {
|
||||
count: vi.fn().mockResolvedValue(1),
|
||||
findMany: vi.fn().mockResolvedValue([{ id: "u1" }]),
|
||||
},
|
||||
};
|
||||
const canonicalPayload = {
|
||||
schemaVersion: BACKUP_SCHEMA_VERSION,
|
||||
@@ -310,8 +352,10 @@ describe("runOffhostBackup", () => {
|
||||
const s3 = makeS3Mock();
|
||||
const upsert = vi.fn();
|
||||
const prisma = {
|
||||
...purgeLedgerMocks(),
|
||||
offhostBackupState: { upsert },
|
||||
user: {
|
||||
count: vi.fn().mockResolvedValue(1),
|
||||
findMany: vi.fn().mockResolvedValue([{ id: "u1" }, { id: "u2" }]),
|
||||
},
|
||||
};
|
||||
@@ -357,10 +401,14 @@ describe("runOffhostBackup", () => {
|
||||
it("still reports a landed object as uploaded when the ledger write fails", async () => {
|
||||
const s3 = makeS3Mock();
|
||||
const prisma = {
|
||||
...purgeLedgerMocks(),
|
||||
offhostBackupState: {
|
||||
upsert: vi.fn().mockRejectedValue(new Error("pool timeout")),
|
||||
},
|
||||
user: { findMany: vi.fn().mockResolvedValue([{ id: "u1" }]) },
|
||||
user: {
|
||||
count: vi.fn().mockResolvedValue(1),
|
||||
findMany: vi.fn().mockResolvedValue([{ id: "u1" }]),
|
||||
},
|
||||
};
|
||||
|
||||
const report = await runOffhostBackup(
|
||||
@@ -382,8 +430,12 @@ describe("runOffhostBackup", () => {
|
||||
it("refuses an account whose object outgrows one multipart upload", async () => {
|
||||
const s3 = makeS3Mock();
|
||||
const prisma = {
|
||||
...purgeLedgerMocks(),
|
||||
offhostBackupState: { upsert: vi.fn() },
|
||||
user: { findMany: vi.fn().mockResolvedValue([{ id: "u1" }]) },
|
||||
user: {
|
||||
count: vi.fn().mockResolvedValue(1),
|
||||
findMany: vi.fn().mockResolvedValue([{ id: "u1" }]),
|
||||
},
|
||||
};
|
||||
mocks.buildFullBackupPayload.mockResolvedValue({
|
||||
payload: {
|
||||
@@ -419,8 +471,12 @@ describe("runOffhostBackup", () => {
|
||||
it("reports the largest object it wrote", async () => {
|
||||
const s3 = makeS3Mock();
|
||||
const prisma = {
|
||||
...purgeLedgerMocks(),
|
||||
offhostBackupState: { upsert: vi.fn() },
|
||||
user: { findMany: vi.fn().mockResolvedValue([{ id: "u1" }]) },
|
||||
user: {
|
||||
count: vi.fn().mockResolvedValue(1),
|
||||
findMany: vi.fn().mockResolvedValue([{ id: "u1" }]),
|
||||
},
|
||||
};
|
||||
const report = await runOffhostBackup(
|
||||
prisma as never,
|
||||
|
||||
@@ -0,0 +1,205 @@
|
||||
/**
|
||||
* Off-host copies leave the bucket with the account.
|
||||
*
|
||||
* The bucket is a map behind the same `S3Like` boundary the real client
|
||||
* implements; the request ledger is an in-memory table. Mutations that must
|
||||
* turn this red: drop the `deleteObject` call (the objects stay), remove the
|
||||
* request on failure too (the refusal case loses the request), or match on the
|
||||
* key prefix instead of the full account pattern (another account's objects,
|
||||
* or the health-check probe, go too).
|
||||
*/
|
||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
|
||||
vi.mock("@/lib/auth/audit", () => ({ auditLog: vi.fn() }));
|
||||
|
||||
import { auditLog } from "@/lib/auth/audit";
|
||||
import {
|
||||
processOffhostPurges,
|
||||
PURGE_ALL_SUBJECTS,
|
||||
requestOffhostPurge,
|
||||
} from "../offhost-purge";
|
||||
|
||||
interface Row {
|
||||
id: string;
|
||||
subjectId: string;
|
||||
reason: string;
|
||||
requestedAt: Date;
|
||||
attempts: number;
|
||||
lastAttemptAt: Date | null;
|
||||
lastFailure: string | null;
|
||||
}
|
||||
|
||||
function makeLedger(existingUsers: string[] = []) {
|
||||
const rows: Row[] = [];
|
||||
let seq = 0;
|
||||
const prisma = {
|
||||
offhostPurgeRequest: {
|
||||
create: vi.fn(
|
||||
async ({ data }: { data: { subjectId: string; reason: string } }) => {
|
||||
const row: Row = {
|
||||
id: `r${++seq}`,
|
||||
subjectId: data.subjectId,
|
||||
reason: data.reason,
|
||||
requestedAt: new Date(Date.now() + seq),
|
||||
attempts: 0,
|
||||
lastAttemptAt: null,
|
||||
lastFailure: null,
|
||||
};
|
||||
rows.push(row);
|
||||
return { id: row.id };
|
||||
},
|
||||
),
|
||||
findMany: vi.fn(async () =>
|
||||
[...rows].sort((a, b) => +a.requestedAt - +b.requestedAt),
|
||||
),
|
||||
delete: vi.fn(async ({ where }: { where: { id: string } }) => {
|
||||
rows.splice(
|
||||
rows.findIndex((r) => r.id === where.id),
|
||||
1,
|
||||
);
|
||||
}),
|
||||
updateMany: vi.fn(
|
||||
async ({
|
||||
where,
|
||||
data,
|
||||
}: {
|
||||
where: { id: { in: string[] } };
|
||||
data: { lastAttemptAt: Date; lastFailure: string };
|
||||
}) => {
|
||||
for (const row of rows) {
|
||||
if (!where.id.in.includes(row.id)) continue;
|
||||
row.attempts += 1;
|
||||
row.lastAttemptAt = data.lastAttemptAt;
|
||||
row.lastFailure = data.lastFailure;
|
||||
}
|
||||
},
|
||||
),
|
||||
},
|
||||
user: {
|
||||
count: vi.fn(async ({ where }: { where: { id: string } }) =>
|
||||
existingUsers.includes(where.id) ? 1 : 0,
|
||||
),
|
||||
},
|
||||
};
|
||||
return { rows, prisma };
|
||||
}
|
||||
|
||||
function makeBucket(keys: string[]) {
|
||||
const store = new Set(keys);
|
||||
return {
|
||||
store,
|
||||
putObject: vi.fn(),
|
||||
putStream: vi.fn(),
|
||||
getObject: vi.fn(),
|
||||
headObject: vi.fn(),
|
||||
listObjects: vi.fn(async (prefix: string) =>
|
||||
[...store].filter((k) => k.startsWith(prefix)).map((key) => ({ key })),
|
||||
),
|
||||
deleteObject: vi.fn(async (key: string) => {
|
||||
store.delete(key);
|
||||
}),
|
||||
};
|
||||
}
|
||||
|
||||
const BUCKET = [
|
||||
"2026-09-01/user-gone.json.enc",
|
||||
"2026-09-02/user-gone.json.enc",
|
||||
"2026-09-02/user-gone2.json.enc",
|
||||
"2026-09-02/user-stays.json.enc",
|
||||
"_healthcheck/1.bin",
|
||||
];
|
||||
|
||||
beforeEach(() => {
|
||||
vi.unstubAllEnvs();
|
||||
vi.clearAllMocks();
|
||||
vi.stubEnv("BACKUP_S3_ENDPOINT", "https://r2.example");
|
||||
vi.stubEnv("BACKUP_S3_BUCKET", "hl-backups");
|
||||
vi.stubEnv("BACKUP_S3_ACCESS_KEY", "AKIA");
|
||||
vi.stubEnv("BACKUP_S3_SECRET_KEY", "secret");
|
||||
vi.stubEnv("BACKUP_ENCRYPTION_KEY", "ab".repeat(32));
|
||||
});
|
||||
|
||||
describe("off-host purge", () => {
|
||||
it("deletes every copy of the account, and nothing else", async () => {
|
||||
const { rows, prisma } = makeLedger();
|
||||
await requestOffhostPurge(prisma as never, "gone", "account_deleted");
|
||||
const bucket = makeBucket(BUCKET);
|
||||
|
||||
const report = await processOffhostPurges(prisma as never, bucket);
|
||||
|
||||
expect([...bucket.store].sort()).toEqual([
|
||||
"2026-09-02/user-gone2.json.enc",
|
||||
"2026-09-02/user-stays.json.enc",
|
||||
"_healthcheck/1.bin",
|
||||
]);
|
||||
expect(report).toMatchObject({
|
||||
completed: 1,
|
||||
failed: 0,
|
||||
objectsDeleted: 2,
|
||||
});
|
||||
expect(rows).toHaveLength(0);
|
||||
// The account is gone, and so is its audit trail: the receipt carries the
|
||||
// reason and the count, not the account.
|
||||
expect(auditLog).toHaveBeenCalledWith(
|
||||
"offhost.backup.purged",
|
||||
expect.objectContaining({
|
||||
userId: null,
|
||||
details: expect.objectContaining({
|
||||
reason: "account_deleted",
|
||||
objectsDeleted: 2,
|
||||
}),
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it("keeps the request, with the bucket's answer, when a delete is refused", async () => {
|
||||
const { rows, prisma } = makeLedger(["gone"]);
|
||||
await requestOffhostPurge(prisma as never, "gone", "data_wiped");
|
||||
const bucket = makeBucket(BUCKET);
|
||||
bucket.deleteObject.mockRejectedValueOnce(new Error("AccessDenied"));
|
||||
|
||||
const report = await processOffhostPurges(prisma as never, bucket);
|
||||
|
||||
expect(report).toMatchObject({ completed: 0, failed: 1 });
|
||||
expect(rows).toHaveLength(1);
|
||||
expect(rows[0]).toMatchObject({ attempts: 1, lastFailure: "AccessDenied" });
|
||||
|
||||
// The next run finishes it, and the account that still exists keeps the
|
||||
// receipt.
|
||||
const again = await processOffhostPurges(prisma as never, bucket);
|
||||
expect(again.completed).toBe(1);
|
||||
expect(rows).toHaveLength(0);
|
||||
expect(auditLog).toHaveBeenCalledWith(
|
||||
"offhost.backup.purged",
|
||||
expect.objectContaining({ userId: "gone" }),
|
||||
);
|
||||
});
|
||||
|
||||
it("clears every account object for the admin wipe", async () => {
|
||||
const { prisma } = makeLedger();
|
||||
await requestOffhostPurge(
|
||||
prisma as never,
|
||||
PURGE_ALL_SUBJECTS,
|
||||
"admin_wipe",
|
||||
);
|
||||
const bucket = makeBucket(BUCKET);
|
||||
await processOffhostPurges(prisma as never, bucket);
|
||||
expect([...bucket.store]).toEqual(["_healthcheck/1.bin"]);
|
||||
});
|
||||
|
||||
it("records nothing on a host without off-host backup", async () => {
|
||||
vi.stubEnv("BACKUP_S3_ENDPOINT", "");
|
||||
const { rows, prisma } = makeLedger();
|
||||
expect(
|
||||
await requestOffhostPurge(prisma as never, "gone", "account_deleted"),
|
||||
).toBe(false);
|
||||
expect(rows).toHaveLength(0);
|
||||
});
|
||||
|
||||
it("does not list the bucket when nothing is pending", async () => {
|
||||
const { prisma } = makeLedger();
|
||||
const bucket = makeBucket(BUCKET);
|
||||
await processOffhostPurges(prisma as never, bucket);
|
||||
expect(bucket.listObjects).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
@@ -144,6 +144,10 @@ export const JOB_FACT_ALLOWLIST: ReadonlySet<string> = new Set([
|
||||
"offhost_backup_oversized",
|
||||
"offhost_backup_total_users",
|
||||
"offhost_backup_uploaded",
|
||||
"offhost_purge_completed",
|
||||
"offhost_purge_failed",
|
||||
"offhost_purge_objects_deleted",
|
||||
"offhost_purge_pending",
|
||||
"outcome",
|
||||
"passes_failed",
|
||||
"per_sample_rows_deleted",
|
||||
|
||||
@@ -27,6 +27,7 @@ import { Readable, Transform } from "node:stream";
|
||||
import { createGunzip, createGzip, gunzipSync, gzipSync } from "node:zlib";
|
||||
import type { PrismaClient } from "@/generated/prisma/client";
|
||||
import { createRawStreamEncryptor, decryptRawStream } from "@/lib/crypto";
|
||||
import { createBackupKeyIdTextScanner } from "@/lib/export/backup-key-ids";
|
||||
import { streamFullBackupJson } from "@/lib/export/full-backup-stream";
|
||||
import { annotate, getEvent } from "@/lib/logging/context";
|
||||
|
||||
@@ -489,6 +490,24 @@ export interface S3Like {
|
||||
prefix: string,
|
||||
): Promise<Array<{ key: string; lastModified?: Date }>>;
|
||||
deleteObject(key: string): Promise<void>;
|
||||
/**
|
||||
* The bucket's lifecycle rule, as far as the credential may read it.
|
||||
* Optional so a test double that has no opinion leaves it out.
|
||||
*/
|
||||
getLifecycle?(): Promise<OffhostLifecycle>;
|
||||
}
|
||||
|
||||
/**
|
||||
* Whether the bucket retires old copies by itself.
|
||||
*
|
||||
* `configured` names the shortest expiry of an enabled rule; `missing` means
|
||||
* the bucket answered that it has no rule; `unknown` means it would not say
|
||||
* (a credential without `s3:GetLifecycleConfiguration`, a target that does not
|
||||
* implement the call, a network failure) and says nothing either way.
|
||||
*/
|
||||
export interface OffhostLifecycle {
|
||||
state: "configured" | "missing" | "unknown";
|
||||
expirationDays: number | null;
|
||||
}
|
||||
|
||||
export async function getS3Client(cfg: OffhostBackupConfig): Promise<S3Like> {
|
||||
@@ -601,9 +620,78 @@ export async function getS3Client(cfg: OffhostBackupConfig): Promise<S3Like> {
|
||||
new mod.DeleteObjectCommand({ Bucket: cfg.bucket, Key: key }),
|
||||
);
|
||||
},
|
||||
getLifecycle: async () => {
|
||||
try {
|
||||
const out = await client.send(
|
||||
new mod.GetBucketLifecycleConfigurationCommand({
|
||||
Bucket: cfg.bucket,
|
||||
}),
|
||||
{ abortSignal: AbortSignal.timeout(5_000) },
|
||||
);
|
||||
const days = (out.Rules ?? [])
|
||||
.filter((rule) => rule.Status === "Enabled")
|
||||
.map((rule) => rule.Expiration?.Days)
|
||||
.filter((d): d is number => typeof d === "number" && d > 0);
|
||||
return days.length > 0
|
||||
? { state: "configured", expirationDays: Math.min(...days) }
|
||||
: { state: "missing", expirationDays: null };
|
||||
} catch (err) {
|
||||
const name = (err as { name?: string; Code?: string }).name;
|
||||
const code = (err as { Code?: string }).Code;
|
||||
if (
|
||||
name === "NoSuchLifecycleConfiguration" ||
|
||||
code === "NoSuchLifecycleConfiguration"
|
||||
) {
|
||||
return { state: "missing", expirationDays: null };
|
||||
}
|
||||
return { state: "unknown", expirationDays: null };
|
||||
}
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
const LIFECYCLE_CACHE_MS = 10 * 60 * 1000;
|
||||
let lifecycleCache: { at: number; value: OffhostLifecycle } | null = null;
|
||||
|
||||
/**
|
||||
* The bucket's lifecycle rule for the admin card, cached for ten minutes so a
|
||||
* page render does not become a bucket call. Never throws.
|
||||
*/
|
||||
export async function probeOffhostLifecycle(
|
||||
s3Override?: S3Like,
|
||||
now: number = Date.now(),
|
||||
): Promise<OffhostLifecycle> {
|
||||
if (
|
||||
!s3Override &&
|
||||
lifecycleCache &&
|
||||
now - lifecycleCache.at < LIFECYCLE_CACHE_MS
|
||||
) {
|
||||
return lifecycleCache.value;
|
||||
}
|
||||
const cfg = loadOffhostConfigSafe();
|
||||
if (!cfg) return { state: "unknown", expirationDays: null };
|
||||
let value: OffhostLifecycle;
|
||||
try {
|
||||
const s3 = s3Override ?? (await getS3Client(cfg));
|
||||
value = s3.getLifecycle
|
||||
? await s3.getLifecycle()
|
||||
: { state: "unknown", expirationDays: null };
|
||||
} catch {
|
||||
value = { state: "unknown", expirationDays: null };
|
||||
}
|
||||
if (!s3Override) lifecycleCache = { at: now, value };
|
||||
return value;
|
||||
}
|
||||
|
||||
/** `loadOffhostConfig`, answering null for a malformed key instead of throwing. */
|
||||
function loadOffhostConfigSafe(): OffhostBackupConfig | null {
|
||||
try {
|
||||
return loadOffhostConfig();
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
interface BackupRunReport {
|
||||
config: { endpoint: string; bucket: string; region: string };
|
||||
uploaded: number;
|
||||
@@ -642,6 +730,30 @@ export interface RunOffhostBackupOptions extends UploadBackupOptions {
|
||||
shouldStop?: () => boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
* Note, per key id, that an object needing it went into the bucket now. Never
|
||||
* fails the account: the object is in the bucket whatever this row says.
|
||||
*/
|
||||
async function recordOffhostKeyUse(
|
||||
prisma: PrismaClient,
|
||||
keyIds: readonly string[],
|
||||
at: Date,
|
||||
): Promise<void> {
|
||||
for (const keyId of keyIds) {
|
||||
try {
|
||||
await prisma.offhostBackupKeyUse.upsert({
|
||||
where: { keyId },
|
||||
create: { keyId, firstWrittenAt: at, lastWrittenAt: at },
|
||||
update: { lastWrittenAt: at },
|
||||
});
|
||||
} catch (err) {
|
||||
getEvent()?.addWarning(
|
||||
`offhost-backup key-use write failed: ${(err as Error).message?.slice(0, 200)}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export async function runOffhostBackup(
|
||||
prisma: PrismaClient,
|
||||
s3Override?: S3Like,
|
||||
@@ -689,24 +801,54 @@ export async function runOffhostBackup(
|
||||
break;
|
||||
}
|
||||
let objectBytes: number | null = null;
|
||||
const objectKey = `${dateKey}/user-${user.id}.json.enc`;
|
||||
const accountStartedAt = new Date();
|
||||
// The key ids the object's content needs, read as it is written: the
|
||||
// admin encryption view uses them to say how long a retired key is still
|
||||
// needed for what is in the bucket.
|
||||
const keyScanner = createBackupKeyIdTextScanner();
|
||||
try {
|
||||
objectBytes = await uploadEncryptedBackup(
|
||||
s3,
|
||||
`${dateKey}/user-${user.id}.json.enc`,
|
||||
objectKey,
|
||||
cfg.encryptionKey,
|
||||
// The same writer the weekly in-database pass uses. The payload
|
||||
// builder was always shared; everything after it was not, which is why
|
||||
// this job kept dying on a record the weekly one had learned to
|
||||
// survive.
|
||||
(write) =>
|
||||
streamFullBackupJson(prisma, user.id, write, {
|
||||
purpose: "disaster-recovery",
|
||||
exportedAt: now,
|
||||
}),
|
||||
streamFullBackupJson(
|
||||
prisma,
|
||||
user.id,
|
||||
async (chunk) => {
|
||||
keyScanner.feed(chunk);
|
||||
await write(chunk);
|
||||
},
|
||||
{
|
||||
purpose: "disaster-recovery",
|
||||
exportedAt: now,
|
||||
},
|
||||
),
|
||||
options,
|
||||
);
|
||||
// The account was deleted or wiped while its copy was being written.
|
||||
// The purge that request started may already have run, so the copy
|
||||
// this run just put there is removed here rather than left behind.
|
||||
const purgedMeanwhile =
|
||||
(await prisma.offhostPurgeRequest.count({
|
||||
where: {
|
||||
subjectId: { in: [user.id, "*"] },
|
||||
requestedAt: { gte: accountStartedAt },
|
||||
},
|
||||
})) > 0 || (await prisma.user.count({ where: { id: user.id } })) === 0;
|
||||
if (purgedMeanwhile) {
|
||||
await s3.deleteObject(objectKey);
|
||||
objectBytes = null;
|
||||
continue;
|
||||
}
|
||||
largestObjectBytes = Math.max(largestObjectBytes, objectBytes);
|
||||
uploaded++;
|
||||
await recordOffhostKeyUse(prisma, keyScanner.keyIds(), new Date());
|
||||
} catch (err) {
|
||||
failed++;
|
||||
if (err instanceof OffhostBackupTooLargeError) oversized++;
|
||||
|
||||
@@ -0,0 +1,270 @@
|
||||
/**
|
||||
* Off-host copies leave the bucket with the account.
|
||||
*
|
||||
* The nightly job writes `<YYYY-MM-DD>/user-<id>.json.enc` for every account,
|
||||
* and until v1.39.3 nothing ever removed one: deleting an account, or wiping
|
||||
* its data, took the rows in this database and left every nightly copy of the
|
||||
* record in the operator's bucket for as long as its lifecycle rule kept them,
|
||||
* or forever where no rule was set. A confirmation that says "permanently
|
||||
* delete" has to mean the off-host copies too.
|
||||
*
|
||||
* How. The deletion writes an `OffhostPurgeRequest` in its own transaction, so
|
||||
* the request exists exactly when the deletion committed. This job then lists
|
||||
* the bucket once, deletes every object of every pending account, and removes
|
||||
* the request when nothing of the account is left. A request the bucket
|
||||
* refuses (a credential without DeleteObject, an outage) stays, counts its
|
||||
* attempts and keeps the bucket's answer, which the admin off-host card shows;
|
||||
* the job runs again every night and after every nightly upload, so a late
|
||||
* copy the upload wrote for an account deleted while it ran is caught too.
|
||||
*
|
||||
* `subjectId` "*" is the admin wipe of every account: every account object
|
||||
* goes. The health-check probes and anything else in the bucket that is not
|
||||
* an account object is never touched.
|
||||
*/
|
||||
import type { Job } from "pg-boss";
|
||||
|
||||
import type { Prisma, PrismaClient } from "@/generated/prisma/client";
|
||||
import { auditLog } from "@/lib/auth/audit";
|
||||
import { getGlobalBoss } from "@/lib/jobs/boss-instance";
|
||||
import { jobDone, jobFailed, type JobOutcome } from "@/lib/jobs/job-outcome";
|
||||
import {
|
||||
getS3Client,
|
||||
loadOffhostConfig,
|
||||
offhostBackupConfigured,
|
||||
type S3Like,
|
||||
} from "@/lib/jobs/offhost-backup";
|
||||
import { reportWorkerError } from "@/lib/jobs/report-worker-error";
|
||||
import { getWorkerPrisma } from "@/lib/jobs/reminder/shared";
|
||||
import { withBackgroundEvent } from "@/lib/logging/background";
|
||||
|
||||
export const OFFHOST_PURGE_QUEUE = "offhost-backup-purge";
|
||||
/** 03:40, after the nightly upload (02:30) has had its hour. */
|
||||
export const OFFHOST_PURGE_CRON = "40 3 * * *";
|
||||
|
||||
/** Every subject, for the admin wipe of every account. */
|
||||
export const PURGE_ALL_SUBJECTS = "*";
|
||||
|
||||
export type OffhostPurgeReason =
|
||||
"account_deleted" | "data_wiped" | "managed_profile_deleted" | "admin_wipe";
|
||||
|
||||
/** An account object: `<date>/user-<id>.json.enc`. */
|
||||
const ACCOUNT_OBJECT = /^\d{4}-\d{2}-\d{2}\/user-(.+)\.json\.enc$/;
|
||||
|
||||
/** Requests one run works through; the next run takes the rest. */
|
||||
const REQUESTS_PER_RUN = 200;
|
||||
|
||||
/** Deletes one run may issue, so a huge bucket cannot run the job forever. */
|
||||
const MAX_DELETES_PER_RUN = 50_000;
|
||||
|
||||
type Db = Pick<Prisma.TransactionClient, "offhostPurgeRequest">;
|
||||
|
||||
/**
|
||||
* Record that `subjectId`'s off-host copies have to go. Call it with the
|
||||
* transaction that deletes the account or wipes its data, so the request and
|
||||
* the deletion commit together. A host without off-host backup configured has
|
||||
* nothing in any bucket this could reach, and records nothing.
|
||||
*/
|
||||
export async function requestOffhostPurge(
|
||||
db: Db,
|
||||
subjectId: string,
|
||||
reason: OffhostPurgeReason,
|
||||
): Promise<boolean> {
|
||||
if (!offhostBackupConfigured()) return false;
|
||||
await db.offhostPurgeRequest.create({
|
||||
data: { subjectId, reason },
|
||||
select: { id: true },
|
||||
});
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* Wake the purge job now rather than at its nightly slot. Best effort: the
|
||||
* request row is what guarantees the purge, and the cron picks it up if the
|
||||
* queue is not reachable from here.
|
||||
*/
|
||||
export async function kickOffhostPurge(): Promise<void> {
|
||||
try {
|
||||
await getGlobalBoss()?.send(
|
||||
OFFHOST_PURGE_QUEUE,
|
||||
{},
|
||||
{
|
||||
singletonKey: OFFHOST_PURGE_QUEUE,
|
||||
retryLimit: 3,
|
||||
retryDelay: 300,
|
||||
},
|
||||
);
|
||||
} catch {
|
||||
// The cron is the backstop.
|
||||
}
|
||||
}
|
||||
|
||||
export interface OffhostPurgeReport {
|
||||
pending: number;
|
||||
completed: number;
|
||||
failed: number;
|
||||
objectsDeleted: number;
|
||||
}
|
||||
|
||||
function trimError(err: unknown): string {
|
||||
return (err instanceof Error ? err.message : String(err)).slice(0, 300);
|
||||
}
|
||||
|
||||
/**
|
||||
* Work through the pending requests: list the bucket once, delete every
|
||||
* object of every pending subject, remove each request whose objects are all
|
||||
* gone. Answers what it did.
|
||||
*/
|
||||
export async function processOffhostPurges(
|
||||
prisma: PrismaClient,
|
||||
s3Override?: S3Like,
|
||||
): Promise<OffhostPurgeReport> {
|
||||
const requests = await prisma.offhostPurgeRequest.findMany({
|
||||
orderBy: { requestedAt: "asc" },
|
||||
take: REQUESTS_PER_RUN,
|
||||
});
|
||||
const report: OffhostPurgeReport = {
|
||||
pending: requests.length,
|
||||
completed: 0,
|
||||
failed: 0,
|
||||
objectsDeleted: 0,
|
||||
};
|
||||
if (requests.length === 0) return report;
|
||||
|
||||
const cfg = loadOffhostConfig();
|
||||
if (!cfg) {
|
||||
// Configuration removed after the requests were written: leave them, an
|
||||
// operator who puts it back gets the purge.
|
||||
return report;
|
||||
}
|
||||
|
||||
const now = new Date();
|
||||
const markFailed = async (ids: string[], err: unknown) => {
|
||||
await prisma.offhostPurgeRequest.updateMany({
|
||||
where: { id: { in: ids } },
|
||||
data: {
|
||||
attempts: { increment: 1 },
|
||||
lastAttemptAt: now,
|
||||
lastFailure: trimError(err),
|
||||
},
|
||||
});
|
||||
report.failed += ids.length;
|
||||
};
|
||||
|
||||
let objects: Array<{ key: string }>;
|
||||
let s3: S3Like;
|
||||
try {
|
||||
s3 = s3Override ?? (await getS3Client(cfg));
|
||||
objects = await s3.listObjects("");
|
||||
} catch (err) {
|
||||
await markFailed(
|
||||
requests.map((r) => r.id),
|
||||
err,
|
||||
);
|
||||
return report;
|
||||
}
|
||||
|
||||
const everything = requests.some((r) => r.subjectId === PURGE_ALL_SUBJECTS);
|
||||
const subjects = new Set(requests.map((r) => r.subjectId));
|
||||
const bySubject = new Map<string, string[]>();
|
||||
for (const { key } of objects) {
|
||||
const match = ACCOUNT_OBJECT.exec(key);
|
||||
if (!match) continue;
|
||||
const subject = match[1];
|
||||
if (!everything && !subjects.has(subject)) continue;
|
||||
const list = bySubject.get(subject) ?? [];
|
||||
list.push(key);
|
||||
bySubject.set(subject, list);
|
||||
}
|
||||
|
||||
// One subject at a time, so a refusal part-way fails only the subjects it
|
||||
// reached, and a request is removed only when its own objects are gone.
|
||||
const failedSubjects = new Map<string, unknown>();
|
||||
let deletes = 0;
|
||||
for (const [subject, keys] of bySubject) {
|
||||
for (const key of keys) {
|
||||
if (deletes >= MAX_DELETES_PER_RUN) {
|
||||
failedSubjects.set(
|
||||
subject,
|
||||
new Error("Deletion budget for one run reached; continues next run"),
|
||||
);
|
||||
break;
|
||||
}
|
||||
try {
|
||||
await s3.deleteObject(key);
|
||||
deletes++;
|
||||
report.objectsDeleted++;
|
||||
} catch (err) {
|
||||
failedSubjects.set(subject, err);
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for (const request of requests) {
|
||||
const failure =
|
||||
request.subjectId === PURGE_ALL_SUBJECTS
|
||||
? failedSubjects.size > 0
|
||||
? [...failedSubjects.values()][0]
|
||||
: null
|
||||
: failedSubjects.get(request.subjectId);
|
||||
if (failure) {
|
||||
await markFailed([request.id], failure);
|
||||
continue;
|
||||
}
|
||||
const deleted =
|
||||
request.subjectId === PURGE_ALL_SUBJECTS
|
||||
? [...bySubject.values()].reduce((sum, keys) => sum + keys.length, 0)
|
||||
: (bySubject.get(request.subjectId)?.length ?? 0);
|
||||
await prisma.offhostPurgeRequest.delete({ where: { id: request.id } });
|
||||
report.completed++;
|
||||
// The account a deleted subject named is gone, and so is its audit
|
||||
// history; the receipt names the reason and the count, not the account.
|
||||
// A wiped account still exists and keeps the receipt as its own.
|
||||
const accountRemains =
|
||||
request.reason === "data_wiped" &&
|
||||
(await prisma.user.count({ where: { id: request.subjectId } })) > 0;
|
||||
await auditLog("offhost.backup.purged", {
|
||||
userId: accountRemains ? request.subjectId : null,
|
||||
actorUserId: null,
|
||||
details: {
|
||||
reason: request.reason,
|
||||
objectsDeleted: deleted,
|
||||
attempts: request.attempts + 1,
|
||||
},
|
||||
});
|
||||
}
|
||||
return report;
|
||||
}
|
||||
|
||||
export async function handleOffhostPurge(
|
||||
jobs: Job<object>[],
|
||||
): Promise<JobOutcome> {
|
||||
void jobs;
|
||||
return withBackgroundEvent("job.offhost_purge", async (evt) => {
|
||||
try {
|
||||
const report = await processOffhostPurges(getWorkerPrisma());
|
||||
evt.addMeta("offhost_purge_pending", report.pending);
|
||||
evt.addMeta("offhost_purge_completed", report.completed);
|
||||
evt.addMeta("offhost_purge_failed", report.failed);
|
||||
evt.addMeta("offhost_purge_objects_deleted", report.objectsDeleted);
|
||||
if (report.failed > 0) {
|
||||
await reportWorkerError(
|
||||
OFFHOST_PURGE_QUEUE,
|
||||
new Error(
|
||||
`${report.failed} off-host deletion request(s) could not be completed; they stay pending and are retried nightly. Check that the bucket credential allows DeleteObject.`,
|
||||
),
|
||||
);
|
||||
}
|
||||
return jobDone({
|
||||
offhost_purge_pending: report.pending,
|
||||
offhost_purge_completed: report.completed,
|
||||
offhost_purge_failed: report.failed,
|
||||
offhost_purge_objects_deleted: report.objectsDeleted,
|
||||
});
|
||||
} catch (err) {
|
||||
evt.addWarning(`offhost-purge failed: ${err}`);
|
||||
await reportWorkerError(OFFHOST_PURGE_QUEUE, err);
|
||||
return jobFailed("offhost purge failed", err);
|
||||
}
|
||||
});
|
||||
}
|
||||
@@ -67,6 +67,11 @@ import {
|
||||
enqueueBootTimeIntakeSlotDedup,
|
||||
type IntakeSlotDedupPayload,
|
||||
} from "@/lib/medications/intake-slot-dedup";
|
||||
import {
|
||||
handleOffhostPurge,
|
||||
OFFHOST_PURGE_CRON,
|
||||
OFFHOST_PURGE_QUEUE,
|
||||
} from "@/lib/jobs/offhost-purge";
|
||||
import {
|
||||
handleRestoreDrill,
|
||||
RESTORE_DRILL_CRON,
|
||||
@@ -353,6 +358,10 @@ const allQueues = [
|
||||
STEP_UP_ELEVATION_CLEANUP_QUEUE,
|
||||
OFFHOST_BACKUP_QUEUE,
|
||||
RESTORE_DRILL_QUEUE,
|
||||
// v1.39.3 — deleting an account or wiping its data takes its off-host
|
||||
// copies out of the bucket. Without this entry the requests would be
|
||||
// written and never worked.
|
||||
OFFHOST_PURGE_QUEUE,
|
||||
// v1.39.1 — an admin's restore of a stored backup, run off the request, and
|
||||
// the boot sweep that re-queues one a stopped worker left running. Without
|
||||
// this entry every restore request would queue a job nobody works.
|
||||
@@ -505,6 +514,9 @@ const schedules: ScheduleEntry[] = [
|
||||
],
|
||||
[OFFHOST_BACKUP_QUEUE, OFFHOST_BACKUP_CRON],
|
||||
[RESTORE_DRILL_QUEUE, RESTORE_DRILL_CRON],
|
||||
// Nightly backstop for the purge the deletion kicks straight away: a
|
||||
// request the bucket refused, or one written while the queue was down.
|
||||
[OFFHOST_PURGE_QUEUE, OFFHOST_PURGE_CRON, cronIsTheRetry],
|
||||
[HOST_METRIC_QUEUE, HOST_METRIC_CRON, cronIsTheRetry],
|
||||
[FEEDBACK_AGGREGATOR_QUEUE, FEEDBACK_AGGREGATOR_CRON, cronIsTheRetry],
|
||||
// v1.4.37 — hourly geo backfill. The helper is idempotent + capped
|
||||
@@ -747,6 +759,12 @@ export async function registerMaintenanceQueues(
|
||||
{ localConcurrency: 1 },
|
||||
handleRestoreDrill,
|
||||
);
|
||||
await createAndWork(
|
||||
boss,
|
||||
OFFHOST_PURGE_QUEUE,
|
||||
{ localConcurrency: 1 },
|
||||
handleOffhostPurge,
|
||||
);
|
||||
// One restore at a time per worker: each holds a transaction over a whole
|
||||
// account and a batch of readings in memory, and two at once on a small host
|
||||
// is how a restore that fits alone would run out of heap.
|
||||
|
||||
@@ -101,6 +101,7 @@ const subsystemSurface = {
|
||||
"step-up-elevation-cleanup": { audience: "system" },
|
||||
"data-backup-offhost": { audience: "system" },
|
||||
"data-restore-drill": { audience: "system" },
|
||||
"offhost-backup-purge": { audience: "system" },
|
||||
"backup-restore": { audience: "system" },
|
||||
"host-metric-sample": { audience: "system" },
|
||||
"feedback-aggregator": { audience: "system" },
|
||||
|
||||
@@ -1,3 +1,7 @@
|
||||
import {
|
||||
kickOffhostPurge,
|
||||
requestOffhostPurge,
|
||||
} from "@/lib/jobs/offhost-purge";
|
||||
import { prisma } from "@/lib/db";
|
||||
import { auditLog } from "@/lib/auth/audit";
|
||||
import type { Prisma } from "@/generated/prisma/client";
|
||||
@@ -209,9 +213,11 @@ export async function deleteManagedProfile(input: {
|
||||
details: { profileId: profile.id },
|
||||
client: tx,
|
||||
});
|
||||
await requestOffhostPurge(tx, profile.id, "managed_profile_deleted");
|
||||
await tx.user.delete({ where: { id: profile.id } });
|
||||
});
|
||||
});
|
||||
await kickOffhostPurge();
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
@@ -66,6 +66,8 @@ export const ERROR_CODE_CATALOGUE: Readonly<Record<string, readonly string[]>> =
|
||||
"auth.token.invalid",
|
||||
],
|
||||
backup: [
|
||||
"backup.foreign_reference",
|
||||
"backup.key.missing",
|
||||
"backup.payload.undecryptable",
|
||||
"backup.restore.active",
|
||||
"backup.section.missing",
|
||||
|
||||
@@ -58,6 +58,25 @@ export interface OffhostBackupOverview {
|
||||
periodHours: number;
|
||||
/** One row per account. Empty when off-host backup is not configured. */
|
||||
rows: OffhostAccountRow[];
|
||||
/**
|
||||
* Deleted or wiped accounts whose copies are still in the bucket. A count
|
||||
* above zero with an error means the bucket refused the deletion (most
|
||||
* often a credential without DeleteObject); the job retries nightly.
|
||||
*/
|
||||
pendingDeletions: {
|
||||
count: number;
|
||||
oldestRequestedAt: string | null;
|
||||
lastFailure: string | null;
|
||||
};
|
||||
/**
|
||||
* Whether the bucket retires old copies by itself. Read from the bucket
|
||||
* (`GetBucketLifecycleConfiguration`), cached for ten minutes; `unknown`
|
||||
* when the credential may not read it.
|
||||
*/
|
||||
lifecycle: {
|
||||
state: "configured" | "missing" | "unknown";
|
||||
expirationDays: number | null;
|
||||
};
|
||||
}
|
||||
|
||||
export interface BackupsList {
|
||||
|
||||
@@ -0,0 +1,126 @@
|
||||
/**
|
||||
* A stored backup records the keys its content needs, and the encryption view
|
||||
* says a retired key is still needed while any backup lists it.
|
||||
*
|
||||
* Both ends and the pipe: the copy is written by the real `storeBackupBlob`,
|
||||
* the key ids are read from the JSON on its way into the pieces, and the
|
||||
* admin status route reads them back. The case the view used to get wrong is
|
||||
* the one asserted: every row rotated, the badge said the old key was safe to
|
||||
* drop, and a backup still needed it.
|
||||
*
|
||||
* Mutations that must turn this red: drop the scanner from `storeBackupBlob`
|
||||
* (nothing is recorded, the view has no key to name), or leave
|
||||
* `retiredKeysStillNeeded` out of `safeToDropRetiredKeys` (the badge says
|
||||
* safe again).
|
||||
*/
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
|
||||
import { _resetCryptoCacheForTests, encrypt } from "@/lib/crypto";
|
||||
|
||||
import { cookieJar, headerJar } from "./mock-next-headers";
|
||||
import { getPrismaClient, truncateAllTables } from "./setup";
|
||||
|
||||
vi.mock("next/headers", async () => {
|
||||
const { cookieJar, headerJar } = await import("./mock-next-headers");
|
||||
return {
|
||||
headers: vi.fn(async () => ({
|
||||
get: (name: string) => headerJar.get(name.toLowerCase()) ?? null,
|
||||
})),
|
||||
cookies: vi.fn(async () => ({
|
||||
get: (name: string) => {
|
||||
const value = cookieJar.get(name);
|
||||
return value ? { name, value } : undefined;
|
||||
},
|
||||
set: (name: string, value: string) => {
|
||||
cookieJar.set(name, value);
|
||||
},
|
||||
delete: (name: string) => {
|
||||
cookieJar.delete(name);
|
||||
},
|
||||
})),
|
||||
};
|
||||
});
|
||||
|
||||
vi.mock("@/lib/db-compat", () => ({
|
||||
ensureDbCompatibility: vi.fn().mockResolvedValue(undefined),
|
||||
}));
|
||||
|
||||
const saved = {
|
||||
key: process.env.ENCRYPTION_KEY,
|
||||
keys: process.env.ENCRYPTION_KEYS,
|
||||
active: process.env.ENCRYPTION_ACTIVE_KEY_ID,
|
||||
};
|
||||
|
||||
function useKeys(active: string) {
|
||||
process.env.ENCRYPTION_KEY = "";
|
||||
process.env.ENCRYPTION_KEYS = JSON.stringify({
|
||||
old: "11".repeat(32),
|
||||
cur: "22".repeat(32),
|
||||
});
|
||||
process.env.ENCRYPTION_ACTIVE_KEY_ID = active;
|
||||
_resetCryptoCacheForTests();
|
||||
}
|
||||
|
||||
beforeEach(async () => {
|
||||
await truncateAllTables(getPrismaClient());
|
||||
cookieJar.clear();
|
||||
headerJar.clear();
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
process.env.ENCRYPTION_KEY = saved.key ?? "";
|
||||
process.env.ENCRYPTION_KEYS = saved.keys ?? "";
|
||||
process.env.ENCRYPTION_ACTIVE_KEY_ID = saved.active ?? "";
|
||||
_resetCryptoCacheForTests();
|
||||
});
|
||||
|
||||
describe("the keys stored backups still need", () => {
|
||||
it("names a retired key a backup's content was written under", async () => {
|
||||
const prisma = getPrismaClient();
|
||||
const admin = await prisma.user.create({
|
||||
data: { username: "keys-admin", email: "k@example.test", role: "ADMIN" },
|
||||
});
|
||||
const session = await prisma.session.create({
|
||||
data: { userId: admin.id, expiresAt: new Date(Date.now() + 60_000) },
|
||||
});
|
||||
cookieJar.set("healthlog_session", session.id);
|
||||
|
||||
const { storeBackupBlob } = await import("@/lib/export/store-backup-blob");
|
||||
// A copy taken before the rotation: its note was written under "old".
|
||||
useKeys("old");
|
||||
const note = Buffer.from(encrypt("before"), "utf8").toString("base64");
|
||||
// The copy itself is stored after the rotation, so its envelope is
|
||||
// sealed under "cur" and only its content needs "old".
|
||||
useKeys("cur");
|
||||
await storeBackupBlob(
|
||||
prisma,
|
||||
{ userId: admin.id, type: "MANUAL_UPLOAD_KEYS" },
|
||||
async (write) => {
|
||||
await write('{"userId":"x","measurements":[{"notesEncrypted":');
|
||||
await write(`"${note}"}]}`);
|
||||
},
|
||||
);
|
||||
const row = await prisma.dataBackup.findFirstOrThrow({
|
||||
where: { type: "MANUAL_UPLOAD_KEYS" },
|
||||
});
|
||||
expect(row.innerKeyIds).toEqual(["old"]);
|
||||
expect(row.innerKeyIdsRecorded).toBe(true);
|
||||
|
||||
const { GET } = await import("@/app/api/admin/encryption/status/route");
|
||||
const res = await GET();
|
||||
expect(res.status).toBe(200);
|
||||
const body = (await res.json()).data as {
|
||||
rotationComplete: boolean;
|
||||
safeToDropRetiredKeys: boolean;
|
||||
backups: {
|
||||
stored: Array<{ keyId: string; copies: number }>;
|
||||
retiredKeysStillNeeded: string[];
|
||||
};
|
||||
};
|
||||
expect(body.backups.stored).toEqual([
|
||||
expect.objectContaining({ keyId: "old", copies: 1 }),
|
||||
]);
|
||||
expect(body.backups.retiredKeysStillNeeded).toEqual(["old"]);
|
||||
expect(body.safeToDropRetiredKeys).toBe(false);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,157 @@
|
||||
/**
|
||||
* Deleting an account, or wiping its data, takes its off-host copies out of
|
||||
* the bucket.
|
||||
*
|
||||
* The bucket is replaced at the client boundary (`getS3Client`) by an
|
||||
* in-memory map; everything above it is real: the route, its transaction, the
|
||||
* request row it commits and the purge job that works it. Mutations that must
|
||||
* turn this red: drop `requestOffhostPurge` from either route (no request is
|
||||
* written and the objects stay), or write the request outside the deletion's
|
||||
* transaction and let it fail (the deletion commits without one).
|
||||
*/
|
||||
import { NextRequest } from "next/server";
|
||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
|
||||
import { cookieJar, headerJar } from "./mock-next-headers";
|
||||
import { getPrismaClient, truncateAllTables } from "./setup";
|
||||
|
||||
const bucket = vi.hoisted(() => new Set<string>());
|
||||
|
||||
vi.mock("@/lib/jobs/offhost-backup", async (importOriginal) => {
|
||||
const actual =
|
||||
await importOriginal<typeof import("@/lib/jobs/offhost-backup")>();
|
||||
return {
|
||||
...actual,
|
||||
getS3Client: vi.fn(async () => ({
|
||||
putObject: vi.fn(),
|
||||
putStream: vi.fn(),
|
||||
getObject: vi.fn(),
|
||||
headObject: vi.fn(),
|
||||
listObjects: vi.fn(async (prefix: string) =>
|
||||
[...bucket].filter((k) => k.startsWith(prefix)).map((key) => ({ key })),
|
||||
),
|
||||
deleteObject: vi.fn(async (key: string) => {
|
||||
bucket.delete(key);
|
||||
}),
|
||||
})),
|
||||
};
|
||||
});
|
||||
|
||||
vi.mock("next/headers", async () => {
|
||||
const { cookieJar, headerJar } = await import("./mock-next-headers");
|
||||
return {
|
||||
headers: vi.fn(async () => ({
|
||||
get: (name: string) => headerJar.get(name.toLowerCase()) ?? null,
|
||||
})),
|
||||
cookies: vi.fn(async () => ({
|
||||
get: (name: string) => {
|
||||
const value = cookieJar.get(name);
|
||||
return value ? { name, value } : undefined;
|
||||
},
|
||||
set: (name: string, value: string) => {
|
||||
cookieJar.set(name, value);
|
||||
},
|
||||
delete: (name: string) => {
|
||||
cookieJar.delete(name);
|
||||
},
|
||||
})),
|
||||
};
|
||||
});
|
||||
|
||||
vi.mock("@/lib/db-compat", () => ({
|
||||
ensureDbCompatibility: vi.fn().mockResolvedValue(undefined),
|
||||
}));
|
||||
|
||||
beforeEach(async () => {
|
||||
await truncateAllTables(getPrismaClient());
|
||||
cookieJar.clear();
|
||||
headerJar.clear();
|
||||
bucket.clear();
|
||||
process.env.BACKUP_S3_ENDPOINT = "https://r2.example";
|
||||
process.env.BACKUP_S3_BUCKET = "hl-backups";
|
||||
process.env.BACKUP_S3_ACCESS_KEY = "AKIA";
|
||||
process.env.BACKUP_S3_SECRET_KEY = "secret";
|
||||
process.env.BACKUP_ENCRYPTION_KEY = "ab".repeat(32);
|
||||
});
|
||||
|
||||
async function seedUserWithSession(username: string) {
|
||||
const prisma = getPrismaClient();
|
||||
const user = await prisma.user.create({
|
||||
data: { username, email: `${username}@example.test` },
|
||||
});
|
||||
const session = await prisma.session.create({
|
||||
data: { userId: user.id, expiresAt: new Date(Date.now() + 3_600_000) },
|
||||
});
|
||||
cookieJar.set("healthlog_session", session.id);
|
||||
return user.id;
|
||||
}
|
||||
|
||||
function seedBucket(userId: string, otherId: string) {
|
||||
for (const day of ["2026-09-01", "2026-09-02", "2026-09-03"]) {
|
||||
bucket.add(`${day}/user-${userId}.json.enc`);
|
||||
bucket.add(`${day}/user-${otherId}.json.enc`);
|
||||
}
|
||||
bucket.add("_healthcheck/1.bin");
|
||||
}
|
||||
|
||||
describe("off-host copies leave with the account", () => {
|
||||
it("account deletion removes every copy of that account", async () => {
|
||||
const prisma = getPrismaClient();
|
||||
const other = await prisma.user.create({
|
||||
data: { username: "neighbour", email: "neighbour@example.test" },
|
||||
});
|
||||
const userId = await seedUserWithSession("leaving");
|
||||
seedBucket(userId, other.id);
|
||||
|
||||
const { DELETE } = await import("@/app/api/settings/account/route");
|
||||
const res = await DELETE(
|
||||
new NextRequest("http://localhost/api/settings/account", {
|
||||
method: "DELETE",
|
||||
headers: { "content-type": "application/json" },
|
||||
body: JSON.stringify({ confirm: "DELETE_ACCOUNT" }),
|
||||
}),
|
||||
);
|
||||
expect(res.status).toBe(200);
|
||||
|
||||
const pending = await prisma.offhostPurgeRequest.findMany();
|
||||
expect(pending.map((r) => [r.subjectId, r.reason])).toEqual([
|
||||
[userId, "account_deleted"],
|
||||
]);
|
||||
|
||||
const { processOffhostPurges } = await import("@/lib/jobs/offhost-purge");
|
||||
const report = await processOffhostPurges(prisma);
|
||||
expect(report).toMatchObject({ completed: 1, objectsDeleted: 3 });
|
||||
expect([...bucket].filter((k) => k.includes(userId))).toEqual([]);
|
||||
expect([...bucket].filter((k) => k.includes(other.id))).toHaveLength(3);
|
||||
expect(bucket.has("_healthcheck/1.bin")).toBe(true);
|
||||
expect(await prisma.offhostPurgeRequest.count()).toBe(0);
|
||||
});
|
||||
|
||||
it("delete-all-data removes every copy and leaves the receipt", async () => {
|
||||
const prisma = getPrismaClient();
|
||||
const other = await prisma.user.create({
|
||||
data: { username: "neighbour2", email: "neighbour2@example.test" },
|
||||
});
|
||||
const userId = await seedUserWithSession("wiping");
|
||||
seedBucket(userId, other.id);
|
||||
|
||||
const { DELETE } = await import("@/app/api/settings/data/route");
|
||||
const res = await DELETE(
|
||||
new NextRequest("http://localhost/api/settings/data", {
|
||||
method: "DELETE",
|
||||
headers: { "content-type": "application/json" },
|
||||
body: JSON.stringify({ confirm: "DELETE" }),
|
||||
}),
|
||||
);
|
||||
expect(res.status).toBe(200);
|
||||
|
||||
const { processOffhostPurges } = await import("@/lib/jobs/offhost-purge");
|
||||
await processOffhostPurges(prisma);
|
||||
expect([...bucket].filter((k) => k.includes(userId))).toEqual([]);
|
||||
expect([...bucket].filter((k) => k.includes(other.id))).toHaveLength(3);
|
||||
const receipt = await prisma.auditLog.findFirst({
|
||||
where: { action: "offhost.backup.purged", userId },
|
||||
});
|
||||
expect(receipt).not.toBeNull();
|
||||
});
|
||||
});
|
||||
Reference in new issue
Block a user