mirror of
https://github.com/MBombeck/HealthLog.git
synced 2026-11-07 04:12:04 +00:00
fix(recovery): the database dumps count in the keys the backups still need
The recovery kit, "Keys the backups still need" and safeToDropRetiredKeys read only the stored copies and the off-host account objects. After a rotation the page called the old key safe to drop while every dump taken before it, the pre-migration dumps that are the way back from an update among them, could only be restored with that key: db-restore refuses a dump whose active key is missing. A dump now counts for the key its rows were written under, the manifest's active key: a dump on this server while it is kept (read from the manifests), an off-host one until the bucket's rule for database/ expires it, or indefinitely when that rule cannot be read. The nightly run records the key on its ledger row (migration 0489, database_backup_runs.active_key_id). The encryption view and the kit list the dumps per key, in every locale, and the rotation and backup runbooks say so.
This commit is contained in:
1 parent
40c310f0ad
commit
26a9efdf52
24 files changed
+422
-45
No files matched your search
@@ -146,8 +146,10 @@ once the lifecycle rule has expired every object written under it.
|
||||
A copy carries the database's ciphertext as stored, so it needs the
|
||||
application keys it was written under, also after a rotation. Keep a retired
|
||||
`ENCRYPTION_KEYS` entry until no copy you may still restore needs it. **Admin →
|
||||
Encryption → Keys the backups still need** lists them. Off-host objects also
|
||||
need `BACKUP_ENCRYPTION_KEY` (or the previous key that wrote them).
|
||||
Encryption → Keys the backups still need** lists them, the database dumps
|
||||
included: a dump needs the key that was active when it was taken, for as long
|
||||
as it is kept on this server or in the bucket. Off-host objects also need
|
||||
`BACKUP_ENCRYPTION_KEY` (or the previous key that wrote them).
|
||||
|
||||
## Size and memory
|
||||
|
||||
|
||||
@@ -137,10 +137,15 @@ under `v1` (the synthetic id assigned to the existing `ENCRYPTION_KEY`).
|
||||
> So before step 4, also check **Admin → Encryption → Keys the backups still
|
||||
> need**. It lists, per key id, the stored copies that need it with the oldest
|
||||
> date, and when the last off-host copy needing it expires under the bucket's
|
||||
> lifecycle rule. Keep the old key in `ENCRYPTION_KEYS` until it is no longer
|
||||
> listed there (the weekly copy replaces itself within a week; delete old
|
||||
> uploaded copies you no longer want, and wait out the off-host retention), or
|
||||
> accept that those copies cannot be restored. Copies written before v1.39.3
|
||||
> lifecycle rule. The database dumps are listed too: a dump is restored only
|
||||
> with the key that was active when it was taken (`db-restore` refuses
|
||||
> without it), so the old key stays listed while a nightly or pre-migration
|
||||
> dump taken under it is on this server, and until the bucket's rule for
|
||||
> `database/` has expired the off-host copies. Keep the old key in
|
||||
> `ENCRYPTION_KEYS` until it is no longer listed there (the weekly copy
|
||||
> replaces itself within a week; delete old uploaded copies you no longer
|
||||
> want, and wait out the dump and off-host retention), or accept that those
|
||||
> copies cannot be restored. Copies written before v1.39.3
|
||||
> did not record their keys and are listed as such: treat them as needing
|
||||
> every key that existed when they were written.
|
||||
>
|
||||
|
||||
+8
-2
@@ -8418,7 +8418,10 @@
|
||||
"columnsHide": "Tabelle ausblenden",
|
||||
"columnsShowAll": "Alle {count} Spalten anzeigen",
|
||||
"columnsShowWithRows": "Nur Spalten mit Zeilen",
|
||||
"columnsNoneWithRows": "Noch keine verschlüsselte Spalte enthält Zeilen."
|
||||
"columnsNoneWithRows": "Noch keine verschlüsselte Spalte enthält Zeilen.",
|
||||
"backupsDumpLocalLine": "Schlüssel {key}: {count} Datenbank-Dumps auf diesem Server wurden unter ihm geschrieben; er wird gebraucht, bis die Aufbewahrung sie entfernt.",
|
||||
"backupsDumpOffhostLine": "Schlüssel {key}: Der neueste Datenbank-Dump, der unter ihm geschrieben wurde, kam am {when} in den Bucket; dort gebraucht bis {until}",
|
||||
"backupsDumpOffhostLineUnknown": "Schlüssel {key}: Der neueste Datenbank-Dump, der unter ihm geschrieben wurde, kam am {when} in den Bucket. Wie lange der Bucket database/ behält, ist unbekannt; behalte den Schlüssel, bis dieser Dump sicher weg ist."
|
||||
},
|
||||
"maintenance": {
|
||||
"title": "Pflege der Messwert-Tabelle",
|
||||
@@ -8604,7 +8607,10 @@
|
||||
"offhostUnknown": "Wird von externen Kopien gebraucht, solange der Bucket sie aufbewahrt.",
|
||||
"backupActive": "Versiegelt jede neue externe Kopie.",
|
||||
"backupPrevious": "Öffnet externe Kopien, die vor dem letzten Schlüsselwechsel geschrieben wurden.",
|
||||
"token": "Signiert Sitzungen, Tokens und Freigabelinks. Ohne ihn werden alle abgemeldet, Daten gehen aber nicht verloren."
|
||||
"token": "Signiert Sitzungen, Tokens und Freigabelinks. Ohne ihn werden alle abgemeldet, Daten gehen aber nicht verloren.",
|
||||
"localDumps": "Wird von {count} Datenbank-Dumps auf diesem Server gebraucht.",
|
||||
"offhostDumpsUntil": "Wird von externen Datenbank-Dumps bis {date} gebraucht.",
|
||||
"offhostDumpsUnknown": "Wird von externen Datenbank-Dumps gebraucht, solange der Bucket sie aufbewahrt."
|
||||
},
|
||||
"source": "Gesetzt in {variable}.",
|
||||
"status": {
|
||||
|
||||
+8
-2
@@ -8418,7 +8418,10 @@
|
||||
"columnsHide": "Hide the table",
|
||||
"columnsShowAll": "Show all {count} columns",
|
||||
"columnsShowWithRows": "Only columns with rows",
|
||||
"columnsNoneWithRows": "No encrypted column holds rows yet."
|
||||
"columnsNoneWithRows": "No encrypted column holds rows yet.",
|
||||
"backupsDumpLocalLine": "Key {key}: {count} database dumps on this server were written under it; it is needed until retention removes them.",
|
||||
"backupsDumpOffhostLine": "Key {key}: the newest database dump written under it went into the bucket on {when}; needed there until {until}",
|
||||
"backupsDumpOffhostLineUnknown": "Key {key}: the newest database dump written under it went into the bucket on {when}. How long the bucket keeps database/ is unknown, so keep the key until you know that dump is gone."
|
||||
},
|
||||
"maintenance": {
|
||||
"title": "Measurement table maintenance",
|
||||
@@ -8604,7 +8607,10 @@
|
||||
"offhostUnknown": "Needed by off-host copies for as long as the bucket keeps them.",
|
||||
"backupActive": "Seals every new off-host copy.",
|
||||
"backupPrevious": "Opens off-host copies written before the last key change.",
|
||||
"token": "Signs sessions, tokens and share links. Without it everyone is signed out, but no data is lost."
|
||||
"token": "Signs sessions, tokens and share links. Without it everyone is signed out, but no data is lost.",
|
||||
"localDumps": "Needed by {count} database dumps on this server.",
|
||||
"offhostDumpsUntil": "Needed by off-host database dumps until {date}.",
|
||||
"offhostDumpsUnknown": "Needed by off-host database dumps for as long as the bucket keeps them."
|
||||
},
|
||||
"source": "Set in {variable}.",
|
||||
"status": {
|
||||
|
||||
+8
-2
@@ -8418,7 +8418,10 @@
|
||||
"columnsHide": "Ocultar la tabla",
|
||||
"columnsShowAll": "Mostrar las {count} columnas",
|
||||
"columnsShowWithRows": "Solo columnas con filas",
|
||||
"columnsNoneWithRows": "Ninguna columna cifrada contiene filas todavía."
|
||||
"columnsNoneWithRows": "Ninguna columna cifrada contiene filas todavía.",
|
||||
"backupsDumpLocalLine": "Clave {key}: {count} volcados de la base de datos en este servidor se escribieron con ella; se necesita hasta que la retención los elimine.",
|
||||
"backupsDumpOffhostLine": "Clave {key}: el volcado de la base de datos más reciente escrito con ella entró en el bucket el {when}; se necesita allí hasta el {until}",
|
||||
"backupsDumpOffhostLineUnknown": "Clave {key}: el volcado de la base de datos más reciente escrito con ella entró en el bucket el {when}. No se sabe cuánto tiempo conserva el bucket database/, así que conserva la clave hasta saber que ese volcado ya no está."
|
||||
},
|
||||
"maintenance": {
|
||||
"title": "Mantenimiento de la tabla de mediciones",
|
||||
@@ -8604,7 +8607,10 @@
|
||||
"offhostUnknown": "La necesitan las copias fuera del host mientras el bucket las conserve.",
|
||||
"backupActive": "Sella cada nueva copia fuera del host.",
|
||||
"backupPrevious": "Abre las copias fuera del host escritas antes del último cambio de clave.",
|
||||
"token": "Firma sesiones, tokens y enlaces para compartir. Sin ella se cierra la sesión de todo el mundo, pero no se pierden datos."
|
||||
"token": "Firma sesiones, tokens y enlaces para compartir. Sin ella se cierra la sesión de todo el mundo, pero no se pierden datos.",
|
||||
"localDumps": "La necesitan {count} volcados de la base de datos en este servidor.",
|
||||
"offhostDumpsUntil": "La necesitan los volcados de la base de datos fuera del host hasta el {date}.",
|
||||
"offhostDumpsUnknown": "La necesitan los volcados de la base de datos fuera del host mientras el bucket los conserve."
|
||||
},
|
||||
"source": "Definida en {variable}.",
|
||||
"status": {
|
||||
|
||||
+8
-2
@@ -8418,7 +8418,10 @@
|
||||
"columnsHide": "Masquer le tableau",
|
||||
"columnsShowAll": "Afficher les {count} colonnes",
|
||||
"columnsShowWithRows": "Seulement les colonnes avec des lignes",
|
||||
"columnsNoneWithRows": "Aucune colonne chiffrée ne contient encore de lignes."
|
||||
"columnsNoneWithRows": "Aucune colonne chiffrée ne contient encore de lignes.",
|
||||
"backupsDumpLocalLine": "Clé {key} : {count} dumps de la base de données sur ce serveur ont été écrits avec elle ; elle est nécessaire jusqu'à ce que la rétention les supprime.",
|
||||
"backupsDumpOffhostLine": "Clé {key} : le dump de la base de données le plus récent écrit avec elle est arrivé dans le bucket le {when} ; nécessaire jusqu'au {until}",
|
||||
"backupsDumpOffhostLineUnknown": "Clé {key} : le dump de la base de données le plus récent écrit avec elle est arrivé dans le bucket le {when}. La durée de conservation de database/ dans le bucket est inconnue : gardez la clé jusqu'à être sûr que ce dump a disparu."
|
||||
},
|
||||
"maintenance": {
|
||||
"title": "Maintenance de la table des mesures",
|
||||
@@ -8604,7 +8607,10 @@
|
||||
"offhostUnknown": "Nécessaire aux copies hors hôte tant que le bucket les conserve.",
|
||||
"backupActive": "Scelle chaque nouvelle copie hors hôte.",
|
||||
"backupPrevious": "Ouvre les copies hors hôte écrites avant le dernier changement de clé.",
|
||||
"token": "Signe les sessions, les jetons et les liens de partage. Sans elle, tout le monde est déconnecté, mais aucune donnée n'est perdue."
|
||||
"token": "Signe les sessions, les jetons et les liens de partage. Sans elle, tout le monde est déconnecté, mais aucune donnée n'est perdue.",
|
||||
"localDumps": "Nécessaire à {count} dumps de la base de données sur ce serveur.",
|
||||
"offhostDumpsUntil": "Nécessaire aux dumps de la base de données hors hôte jusqu'au {date}.",
|
||||
"offhostDumpsUnknown": "Nécessaire aux dumps de la base de données hors hôte tant que le bucket les conserve."
|
||||
},
|
||||
"source": "Définie dans {variable}.",
|
||||
"status": {
|
||||
|
||||
+8
-2
@@ -8418,7 +8418,10 @@
|
||||
"columnsHide": "Nascondi la tabella",
|
||||
"columnsShowAll": "Mostra tutte le {count} colonne",
|
||||
"columnsShowWithRows": "Solo colonne con righe",
|
||||
"columnsNoneWithRows": "Nessuna colonna cifrata contiene ancora righe."
|
||||
"columnsNoneWithRows": "Nessuna colonna cifrata contiene ancora righe.",
|
||||
"backupsDumpLocalLine": "Chiave {key}: {count} dump del database su questo server sono stati scritti con essa; serve finché la conservazione non li rimuove.",
|
||||
"backupsDumpOffhostLine": "Chiave {key}: il dump del database più recente scritto con essa è entrato nel bucket il {when}; serve lì fino al {until}",
|
||||
"backupsDumpOffhostLineUnknown": "Chiave {key}: il dump del database più recente scritto con essa è entrato nel bucket il {when}. Non si sa per quanto il bucket conservi database/, quindi conserva la chiave finché non sai che quel dump non c'è più."
|
||||
},
|
||||
"maintenance": {
|
||||
"title": "Manutenzione della tabella delle misurazioni",
|
||||
@@ -8604,7 +8607,10 @@
|
||||
"offhostUnknown": "Serve alle copie fuori host finché il bucket le conserva.",
|
||||
"backupActive": "Sigilla ogni nuova copia fuori host.",
|
||||
"backupPrevious": "Apre le copie fuori host scritte prima dell'ultimo cambio di chiave.",
|
||||
"token": "Firma sessioni, token e link di condivisione. Senza di essa tutti vengono disconnessi, ma nessun dato va perso."
|
||||
"token": "Firma sessioni, token e link di condivisione. Senza di essa tutti vengono disconnessi, ma nessun dato va perso.",
|
||||
"localDumps": "Serve a {count} dump del database su questo server.",
|
||||
"offhostDumpsUntil": "Serve ai dump del database fuori host fino al {date}.",
|
||||
"offhostDumpsUnknown": "Serve ai dump del database fuori host finché il bucket li conserva."
|
||||
},
|
||||
"source": "Impostata in {variable}.",
|
||||
"status": {
|
||||
|
||||
+8
-2
@@ -8393,7 +8393,10 @@
|
||||
"columnsHide": "표 숨기기",
|
||||
"columnsShowAll": "열 {count}개 모두 보기",
|
||||
"columnsShowWithRows": "행이 있는 열만",
|
||||
"columnsNoneWithRows": "아직 행이 있는 암호화 열이 없어요."
|
||||
"columnsNoneWithRows": "아직 행이 있는 암호화 열이 없어요.",
|
||||
"backupsDumpLocalLine": "키 {key}: 이 서버의 데이터베이스 덤프 {count}개가 이 키로 작성되었습니다. 보존 정책이 덤프를 지울 때까지 필요합니다.",
|
||||
"backupsDumpOffhostLine": "키 {key}: 이 키로 작성된 최신 데이터베이스 덤프가 {when}에 버킷에 저장되었습니다. {until}까지 필요합니다",
|
||||
"backupsDumpOffhostLineUnknown": "키 {key}: 이 키로 작성된 최신 데이터베이스 덤프가 {when}에 버킷에 저장되었습니다. 버킷이 database/를 얼마나 보관하는지 알 수 없으므로 그 덤프가 사라진 것을 확인할 때까지 키를 유지하세요."
|
||||
},
|
||||
"maintenance": {
|
||||
"title": "측정값 테이블 유지 관리",
|
||||
@@ -8604,7 +8607,10 @@
|
||||
"offhostUnknown": "버킷이 외부 보관 사본을 보관하는 동안 필요합니다.",
|
||||
"backupActive": "새 외부 보관 사본을 모두 봉인합니다.",
|
||||
"backupPrevious": "마지막 키 변경 전에 기록된 외부 보관 사본을 엽니다.",
|
||||
"token": "세션, 토큰, 공유 링크에 서명합니다. 이 키가 없으면 모두 로그아웃되지만 데이터는 잃지 않습니다."
|
||||
"token": "세션, 토큰, 공유 링크에 서명합니다. 이 키가 없으면 모두 로그아웃되지만 데이터는 잃지 않습니다.",
|
||||
"localDumps": "이 서버의 데이터베이스 덤프 {count}개에 필요합니다.",
|
||||
"offhostDumpsUntil": "{date}까지 외부 보관 데이터베이스 덤프에 필요합니다.",
|
||||
"offhostDumpsUnknown": "버킷이 외부 보관 데이터베이스 덤프를 보관하는 동안 필요합니다."
|
||||
},
|
||||
"source": "{variable}에 설정됨.",
|
||||
"status": {
|
||||
|
||||
+8
-2
@@ -8418,7 +8418,10 @@
|
||||
"columnsHide": "Tabel verbergen",
|
||||
"columnsShowAll": "Alle {count} kolommen tonen",
|
||||
"columnsShowWithRows": "Alleen kolommen met rijen",
|
||||
"columnsNoneWithRows": "Nog geen enkele versleutelde kolom bevat rijen."
|
||||
"columnsNoneWithRows": "Nog geen enkele versleutelde kolom bevat rijen.",
|
||||
"backupsDumpLocalLine": "Sleutel {key}: {count} databasedumps op deze server zijn onder deze sleutel geschreven; hij is nodig tot de bewaartermijn ze verwijdert.",
|
||||
"backupsDumpOffhostLine": "Sleutel {key}: de nieuwste databasedump onder deze sleutel is op {when} in de bucket gezet; daar nodig tot {until}",
|
||||
"backupsDumpOffhostLineUnknown": "Sleutel {key}: de nieuwste databasedump onder deze sleutel is op {when} in de bucket gezet. Hoe lang de bucket database/ bewaart, is onbekend, dus bewaar de sleutel tot je zeker weet dat die dump weg is."
|
||||
},
|
||||
"maintenance": {
|
||||
"title": "Onderhoud van de metingentabel",
|
||||
@@ -8604,7 +8607,10 @@
|
||||
"offhostUnknown": "Nodig voor externe kopieën zolang de bucket ze bewaart.",
|
||||
"backupActive": "Verzegelt elke nieuwe externe kopie.",
|
||||
"backupPrevious": "Opent externe kopieën die vóór de laatste sleutelwissel zijn geschreven.",
|
||||
"token": "Ondertekent sessies, tokens en deellinks. Zonder deze sleutel wordt iedereen afgemeld, maar gaan er geen gegevens verloren."
|
||||
"token": "Ondertekent sessies, tokens en deellinks. Zonder deze sleutel wordt iedereen afgemeld, maar gaan er geen gegevens verloren.",
|
||||
"localDumps": "Nodig voor {count} databasedumps op deze server.",
|
||||
"offhostDumpsUntil": "Nodig voor externe databasedumps tot {date}.",
|
||||
"offhostDumpsUnknown": "Nodig voor externe databasedumps zolang de bucket ze bewaart."
|
||||
},
|
||||
"source": "Ingesteld in {variable}.",
|
||||
"status": {
|
||||
|
||||
+8
-2
@@ -8418,7 +8418,10 @@
|
||||
"columnsHide": "Ukryj tabelę",
|
||||
"columnsShowAll": "Pokaż wszystkie kolumny ({count})",
|
||||
"columnsShowWithRows": "Tylko kolumny z wierszami",
|
||||
"columnsNoneWithRows": "Żadna zaszyfrowana kolumna nie zawiera jeszcze wierszy."
|
||||
"columnsNoneWithRows": "Żadna zaszyfrowana kolumna nie zawiera jeszcze wierszy.",
|
||||
"backupsDumpLocalLine": "Klucz {key}: {count} zrzutów bazy danych na tym serwerze zapisano pod nim; jest potrzebny, dopóki retencja ich nie usunie.",
|
||||
"backupsDumpOffhostLine": "Klucz {key}: najnowszy zrzut bazy danych zapisany pod nim trafił do bucketu {when}; potrzebny tam do {until}",
|
||||
"backupsDumpOffhostLineUnknown": "Klucz {key}: najnowszy zrzut bazy danych zapisany pod nim trafił do bucketu {when}. Nie wiadomo, jak długo bucket przechowuje database/, więc zachowaj klucz, dopóki nie masz pewności, że tego zrzutu już nie ma."
|
||||
},
|
||||
"maintenance": {
|
||||
"title": "Konserwacja tabeli pomiarów",
|
||||
@@ -8604,7 +8607,10 @@
|
||||
"offhostUnknown": "Potrzebny kopiom poza hostem, dopóki bucket je przechowuje.",
|
||||
"backupActive": "Pieczętuje każdą nową kopię poza hostem.",
|
||||
"backupPrevious": "Otwiera kopie poza hostem zapisane przed ostatnią zmianą klucza.",
|
||||
"token": "Podpisuje sesje, tokeny i linki udostępniania. Bez niego wszyscy zostaną wylogowani, ale żadne dane nie przepadną."
|
||||
"token": "Podpisuje sesje, tokeny i linki udostępniania. Bez niego wszyscy zostaną wylogowani, ale żadne dane nie przepadną.",
|
||||
"localDumps": "Potrzebny zrzutom bazy danych na tym serwerze: {count}.",
|
||||
"offhostDumpsUntil": "Potrzebny zrzutom bazy danych poza hostem do {date}.",
|
||||
"offhostDumpsUnknown": "Potrzebny zrzutom bazy danych poza hostem, dopóki bucket je przechowuje."
|
||||
},
|
||||
"source": "Ustawiony w {variable}.",
|
||||
"status": {
|
||||
|
||||
+8
-2
@@ -8418,7 +8418,10 @@
|
||||
"columnsHide": "Ocultar a tabela",
|
||||
"columnsShowAll": "Mostrar todas as {count} colunas",
|
||||
"columnsShowWithRows": "Somente colunas com linhas",
|
||||
"columnsNoneWithRows": "Nenhuma coluna criptografada contém linhas ainda."
|
||||
"columnsNoneWithRows": "Nenhuma coluna criptografada contém linhas ainda.",
|
||||
"backupsDumpLocalLine": "Chave {key}: {count} dumps do banco de dados neste servidor foram gravados com ela; ela é necessária até que a retenção os remova.",
|
||||
"backupsDumpOffhostLine": "Chave {key}: o dump do banco de dados mais recente gravado com ela foi para o bucket em {when}; necessária lá até {until}",
|
||||
"backupsDumpOffhostLineUnknown": "Chave {key}: o dump do banco de dados mais recente gravado com ela foi para o bucket em {when}. Não se sabe por quanto tempo o bucket mantém database/, então mantenha a chave até saber que aquele dump se foi."
|
||||
},
|
||||
"maintenance": {
|
||||
"title": "Manutenção da tabela de medições",
|
||||
@@ -8604,7 +8607,10 @@
|
||||
"offhostUnknown": "Necessária para as cópias fora do host enquanto o bucket as mantiver.",
|
||||
"backupActive": "Sela cada nova cópia fora do host.",
|
||||
"backupPrevious": "Abre as cópias fora do host gravadas antes da última troca de chave.",
|
||||
"token": "Assina sessões, tokens e links de compartilhamento. Sem ela, todos são desconectados, mas nenhum dado é perdido."
|
||||
"token": "Assina sessões, tokens e links de compartilhamento. Sem ela, todos são desconectados, mas nenhum dado é perdido.",
|
||||
"localDumps": "Necessária para {count} dumps do banco de dados neste servidor.",
|
||||
"offhostDumpsUntil": "Necessária para os dumps do banco de dados fora do host até {date}.",
|
||||
"offhostDumpsUnknown": "Necessária para os dumps do banco de dados fora do host enquanto o bucket os mantiver."
|
||||
},
|
||||
"source": "Definida em {variable}.",
|
||||
"status": {
|
||||
|
||||
@@ -0,0 +1,18 @@
|
||||
-- The nightly database dump records the application key its rows were
|
||||
-- written under (the manifest's active key id). A dump is restored only with
|
||||
-- that key, so the recovery kit and "keys the backups still need" list it for
|
||||
-- as long as the off-host copy of the dump may still be in the bucket.
|
||||
--
|
||||
-- Nullable, no default, no backfill: a run recorded before this column reads
|
||||
-- as "key not recorded", and its dump ages out of the bucket like any other.
|
||||
-- Reversibility: dropping the column restores the previous schema.
|
||||
--
|
||||
-- @retry-on-lock-timeout
|
||||
|
||||
DO $migration$
|
||||
BEGIN
|
||||
PERFORM set_config('lock_timeout', '5s', true);
|
||||
ALTER TABLE "database_backup_runs"
|
||||
ADD COLUMN IF NOT EXISTS "active_key_id" TEXT;
|
||||
END
|
||||
$migration$;
|
||||
@@ -6032,6 +6032,11 @@ model DatabaseBackupRun {
|
||||
offhostResult String @map("offhost_result")
|
||||
/// Why the run failed or was skipped, trimmed. Never key material.
|
||||
error String?
|
||||
/// The application key id the dump's rows were written under (its
|
||||
/// manifest's active key). A restore refuses without it. Reader:
|
||||
/// `readBackupKeyNeeds`, which keeps the key in the recovery kit while the
|
||||
/// off-host copy of this dump may still be in the bucket.
|
||||
activeKeyId String? @map("active_key_id")
|
||||
|
||||
@@index([startedAt])
|
||||
@@map("database_backup_runs")
|
||||
|
||||
@@ -3975,6 +3975,7 @@ export async function seed() {
|
||||
stored: [],
|
||||
unrecorded: { copies: 0, oldestAt: null },
|
||||
offhost: [],
|
||||
dumps: [],
|
||||
offhostExpirationDays: null,
|
||||
retiredKeysStillNeeded: [],
|
||||
},
|
||||
|
||||
@@ -44,6 +44,7 @@ function statusWith(configuredKeyCount: number) {
|
||||
stored: [],
|
||||
unrecorded: { copies: 0, oldestAt: null },
|
||||
offhost: [],
|
||||
dumps: [],
|
||||
offhostExpirationDays: null,
|
||||
retiredKeysStillNeeded: [],
|
||||
},
|
||||
|
||||
@@ -40,6 +40,8 @@ const need = (over: Partial<KitKey["need"]> = {}): KitKey["need"] => ({
|
||||
configured: true,
|
||||
storedCopies: 0,
|
||||
offhostUntil: null,
|
||||
localDumps: 0,
|
||||
offhostDumpsUntil: null,
|
||||
...over,
|
||||
});
|
||||
|
||||
|
||||
@@ -88,6 +88,11 @@ interface EncryptionStatus {
|
||||
lastWrittenAt: string;
|
||||
neededUntil: string | null;
|
||||
}>;
|
||||
dumps: Array<{
|
||||
keyId: string;
|
||||
local: number;
|
||||
offhost: { lastWrittenAt: string; neededUntil: string | null } | null;
|
||||
}>;
|
||||
offhostExpirationDays: number | null;
|
||||
retiredKeysStillNeeded: string[];
|
||||
};
|
||||
@@ -306,9 +311,39 @@ export function EncryptionSection() {
|
||||
})}
|
||||
</li>
|
||||
))}
|
||||
{s.backups.dumps.map((row) => (
|
||||
<li key={`dump-${row.keyId}`}>
|
||||
{[
|
||||
row.local > 0
|
||||
? t("admin.section.encryption.backupsDumpLocalLine", {
|
||||
key: row.keyId,
|
||||
count: row.local,
|
||||
})
|
||||
: null,
|
||||
row.offhost
|
||||
? row.offhost.neededUntil
|
||||
? t("admin.section.encryption.backupsDumpOffhostLine", {
|
||||
key: row.keyId,
|
||||
when: fmt.dateTime(row.offhost.lastWrittenAt),
|
||||
until: fmt.dateTime(row.offhost.neededUntil),
|
||||
})
|
||||
: t(
|
||||
"admin.section.encryption.backupsDumpOffhostLineUnknown",
|
||||
{
|
||||
key: row.keyId,
|
||||
when: fmt.dateTime(row.offhost.lastWrittenAt),
|
||||
},
|
||||
)
|
||||
: null,
|
||||
]
|
||||
.filter((line): line is string => line !== null)
|
||||
.join(" ")}
|
||||
</li>
|
||||
))}
|
||||
{s.backups.stored.length === 0 &&
|
||||
s.backups.unrecorded.copies === 0 &&
|
||||
s.backups.offhost.length === 0 ? (
|
||||
s.backups.offhost.length === 0 &&
|
||||
s.backups.dumps.length === 0 ? (
|
||||
<li className="text-muted-foreground">
|
||||
{t("admin.section.encryption.backupsNone")}
|
||||
</li>
|
||||
|
||||
@@ -138,6 +138,20 @@ function KeyNeeds({ k }: { k: KitKey }) {
|
||||
}),
|
||||
);
|
||||
}
|
||||
if (k.need.localDumps > 0) {
|
||||
lines.push(
|
||||
t("admin.recoveryKit.need.localDumps", { count: k.need.localDumps }),
|
||||
);
|
||||
}
|
||||
if (k.need.offhostDumpsUntil === "unknown") {
|
||||
lines.push(t("admin.recoveryKit.need.offhostDumpsUnknown"));
|
||||
} else if (k.need.offhostDumpsUntil) {
|
||||
lines.push(
|
||||
t("admin.recoveryKit.need.offhostDumpsUntil", {
|
||||
date: fmt.date(new Date(k.need.offhostDumpsUntil)),
|
||||
}),
|
||||
);
|
||||
}
|
||||
}
|
||||
lines.push(t("admin.recoveryKit.source", { variable: k.source }));
|
||||
return <p className="text-muted-foreground text-xs">{lines.join(" ")}</p>;
|
||||
|
||||
@@ -156,6 +156,7 @@ export async function runDatabaseBackup(
|
||||
});
|
||||
|
||||
let scratch: string | null = null;
|
||||
let activeKeyId: string | null = null;
|
||||
try {
|
||||
if (!databaseBackupEnabled(env)) {
|
||||
report.status = "skipped";
|
||||
@@ -182,6 +183,7 @@ export async function runDatabaseBackup(
|
||||
deadline: options.deadline,
|
||||
});
|
||||
if (keepLocal) report.local = "written";
|
||||
activeKeyId = dump.manifest.keys.encryption.activeKeyId;
|
||||
report.fileName = basename(dump.dumpPath);
|
||||
report.dumpBytes = dump.manifest.dump.bytes;
|
||||
|
||||
@@ -233,6 +235,7 @@ export async function runDatabaseBackup(
|
||||
localResult: report.local,
|
||||
offhostResult: report.offhost,
|
||||
error: report.error,
|
||||
activeKeyId,
|
||||
},
|
||||
});
|
||||
await prisma.databaseBackupRun.deleteMany({
|
||||
|
||||
@@ -31,6 +31,7 @@ const NO_NEEDS: BackupKeyNeeds = {
|
||||
stored: [],
|
||||
unrecorded: { copies: 0, oldestAt: null },
|
||||
offhost: [],
|
||||
dumps: [],
|
||||
offhostExpirationDays: null,
|
||||
retiredKeysStillNeeded: [],
|
||||
};
|
||||
@@ -76,6 +77,43 @@ describe("which keys the kit lists", () => {
|
||||
});
|
||||
});
|
||||
|
||||
// A dump is restored only with the key its rows were written under; with
|
||||
// it out of the kit, a rotation read as finished while every dump from
|
||||
// before it, the pre-migration ones among them, still needed the old key.
|
||||
it("keeps a retired key while a database dump written under it exists", () => {
|
||||
const local: BackupKeyNeeds = {
|
||||
...NO_NEEDS,
|
||||
dumps: [{ keyId: "v1", local: 2, offhost: null }],
|
||||
};
|
||||
expect(neededKeyIds(local, NOW)).toEqual(new Set(["v1"]));
|
||||
const [v1] = appKeys(["v2"], "v2", local);
|
||||
expect(v1).toMatchObject({
|
||||
id: "v1",
|
||||
problem: "not_configured",
|
||||
need: { localDumps: 2, offhostDumpsUntil: null },
|
||||
});
|
||||
|
||||
const offhost = (neededUntil: string | null): BackupKeyNeeds => ({
|
||||
...NO_NEEDS,
|
||||
dumps: [
|
||||
{
|
||||
keyId: "v1",
|
||||
local: 0,
|
||||
offhost: { lastWrittenAt: NOW.toISOString(), neededUntil },
|
||||
},
|
||||
],
|
||||
});
|
||||
const later = new Date(NOW.getTime() + 3 * DAY).toISOString();
|
||||
expect(neededKeyIds(offhost(later), NOW).has("v1")).toBe(true);
|
||||
expect(appKeys(["v1", "v2"], "v2", offhost(later))[0].need).toMatchObject({
|
||||
offhostDumpsUntil: later,
|
||||
});
|
||||
// Unknown retention keeps it; an expiry that has passed lets it go.
|
||||
expect(neededKeyIds(offhost(null), NOW).has("v1")).toBe(true);
|
||||
const earlier = new Date(NOW.getTime() - DAY).toISOString();
|
||||
expect(neededKeyIds(offhost(earlier), NOW).has("v1")).toBe(false);
|
||||
});
|
||||
|
||||
it("keeps an off-host key while objects inside the retention need it", () => {
|
||||
const row = (neededUntil: string | null) => ({
|
||||
...NO_NEEDS,
|
||||
|
||||
@@ -10,10 +10,25 @@
|
||||
* bucket the last night an object needing it went in, plus how long the
|
||||
* bucket's lifecycle rule keeps objects after that.
|
||||
*
|
||||
* The database dumps count too. A dump is restored only with the key its rows
|
||||
* were written under (its manifest's active key; `db-restore` refuses without
|
||||
* it), so that key is needed for as long as the dump exists: a dump kept on
|
||||
* this server until retention removes it, an off-host one until the bucket's
|
||||
* rule for `database/` expires it. Without them, a rotation read as safe to
|
||||
* finish a week later while every dump from before it, the pre-migration
|
||||
* dumps that are the way back from an update among them, still needed the
|
||||
* retired key.
|
||||
*
|
||||
* Two readers ask it: the admin encryption view, which shows the table, and
|
||||
* the recovery kit, which lists every key a restore needs. One definition, so
|
||||
* the kit cannot drop a key the encryption view still calls needed.
|
||||
*/
|
||||
import {
|
||||
isGoodSet,
|
||||
listDumpSets,
|
||||
readManifestFile,
|
||||
resolveBackupDir,
|
||||
} from "@/lib/database-backup/dump-core.mjs";
|
||||
import { prisma } from "@/lib/db";
|
||||
import {
|
||||
offhostBackupConfigured,
|
||||
@@ -32,50 +47,109 @@ export interface BackupKeyNeeds {
|
||||
/** When the last object needing this key expires; null when unknown. */
|
||||
neededUntil: string | null;
|
||||
}>;
|
||||
/**
|
||||
* Per key id, the database dumps written under it: how many are kept on
|
||||
* this server, and the newest one that went off-host with when the bucket
|
||||
* expires it (`neededUntil` null when that cannot be read).
|
||||
*/
|
||||
dumps: Array<{
|
||||
keyId: string;
|
||||
local: number;
|
||||
offhost: { lastWrittenAt: string; neededUntil: string | null } | null;
|
||||
}>;
|
||||
/** The bucket's expiry, when it has one the credential may read. */
|
||||
offhostExpirationDays: number | null;
|
||||
/** Retired key ids some backup still needs (never the active one). */
|
||||
retiredKeysStillNeeded: string[];
|
||||
}
|
||||
|
||||
/**
|
||||
* The active key of every good dump in `dir`, one entry per dump. A manifest
|
||||
* that cannot be read names no key; `db-restore` refuses such a dump anyway.
|
||||
*/
|
||||
export async function localDumpKeyIds(dir: string): Promise<string[]> {
|
||||
const keys: string[] = [];
|
||||
for (const set of listDumpSets(dir).filter(isGoodSet)) {
|
||||
try {
|
||||
const manifest = await readManifestFile(set.manifest as string);
|
||||
const id = manifest.keys.encryption.activeKeyId;
|
||||
if (id) keys.push(id);
|
||||
} catch {
|
||||
// Unreadable: no key to name.
|
||||
}
|
||||
}
|
||||
return keys;
|
||||
}
|
||||
|
||||
export async function readBackupKeyNeeds(
|
||||
activeKeyId: string,
|
||||
now: Date,
|
||||
options: { dumpDir?: string } = {},
|
||||
): Promise<BackupKeyNeeds> {
|
||||
const [stored, unrecorded, offhostRows] = await Promise.all([
|
||||
prisma.$queryRaw<
|
||||
Array<{ key_id: string; copies: number; oldest_at: Date }>
|
||||
>`
|
||||
const [stored, unrecorded, offhostRows, dumpUploads, localDumpKeys] =
|
||||
await Promise.all([
|
||||
prisma.$queryRaw<
|
||||
Array<{ key_id: string; copies: number; oldest_at: Date }>
|
||||
>`
|
||||
SELECT k AS key_id, count(*)::int AS copies, min(created_at) AS oldest_at
|
||||
FROM data_backups, unnest(inner_key_ids) AS k
|
||||
WHERE inner_key_ids_recorded
|
||||
GROUP BY k
|
||||
ORDER BY k
|
||||
`,
|
||||
prisma.dataBackup.aggregate({
|
||||
where: { innerKeyIdsRecorded: false },
|
||||
_count: { _all: true },
|
||||
_min: { createdAt: true },
|
||||
}),
|
||||
prisma.offhostBackupKeyUse.findMany({ orderBy: { keyId: "asc" } }),
|
||||
]);
|
||||
prisma.dataBackup.aggregate({
|
||||
where: { innerKeyIdsRecorded: false },
|
||||
_count: { _all: true },
|
||||
_min: { createdAt: true },
|
||||
}),
|
||||
prisma.offhostBackupKeyUse.findMany({ orderBy: { keyId: "asc" } }),
|
||||
prisma.databaseBackupRun.groupBy({
|
||||
by: ["activeKeyId"],
|
||||
where: { offhostResult: "uploaded", activeKeyId: { not: null } },
|
||||
_max: { startedAt: true },
|
||||
}),
|
||||
localDumpKeyIds(options.dumpDir ?? resolveBackupDir()),
|
||||
]);
|
||||
const lifecycle =
|
||||
offhostRows.length > 0 && offhostBackupConfigured()
|
||||
(offhostRows.length > 0 || dumpUploads.length > 0) &&
|
||||
offhostBackupConfigured()
|
||||
? await probeOffhostLifecycle()
|
||||
: null;
|
||||
const expirationDays =
|
||||
lifecycle?.state === "configured" ? lifecycle.expirationDays : null;
|
||||
const dumpExpirationDays =
|
||||
lifecycle?.database.state === "configured"
|
||||
? lifecycle.database.expirationDays
|
||||
: null;
|
||||
const expiresAfter = (writtenAt: Date, days: number | null) =>
|
||||
days === null
|
||||
? null
|
||||
: new Date(writtenAt.getTime() + (days + 1) * 86_400_000).toISOString();
|
||||
|
||||
const dumpKeyIds = new Set([
|
||||
...localDumpKeys,
|
||||
...dumpUploads.map((row) => row.activeKeyId as string),
|
||||
]);
|
||||
const dumps = [...dumpKeyIds].sort().map((keyId) => {
|
||||
const upload = dumpUploads.find((row) => row.activeKeyId === keyId);
|
||||
const writtenAt = upload?._max.startedAt ?? null;
|
||||
return {
|
||||
keyId,
|
||||
local: localDumpKeys.filter((id) => id === keyId).length,
|
||||
offhost: writtenAt
|
||||
? {
|
||||
lastWrittenAt: writtenAt.toISOString(),
|
||||
neededUntil: expiresAfter(writtenAt, dumpExpirationDays),
|
||||
}
|
||||
: null,
|
||||
};
|
||||
});
|
||||
|
||||
const offhost = offhostRows.map((row) => ({
|
||||
keyId: row.keyId,
|
||||
firstWrittenAt: row.firstWrittenAt.toISOString(),
|
||||
lastWrittenAt: row.lastWrittenAt.toISOString(),
|
||||
neededUntil:
|
||||
expirationDays === null
|
||||
? null
|
||||
: new Date(
|
||||
row.lastWrittenAt.getTime() + (expirationDays + 1) * 86_400_000,
|
||||
).toISOString(),
|
||||
neededUntil: expiresAfter(row.lastWrittenAt, expirationDays),
|
||||
}));
|
||||
|
||||
const needs: BackupKeyNeeds = {
|
||||
@@ -89,6 +163,7 @@ export async function readBackupKeyNeeds(
|
||||
oldestAt: unrecorded._min.createdAt?.toISOString() ?? null,
|
||||
},
|
||||
offhost,
|
||||
dumps,
|
||||
offhostExpirationDays: expirationDays,
|
||||
retiredKeysStillNeeded: [],
|
||||
};
|
||||
|
||||
@@ -82,6 +82,10 @@ export interface KitKeyNeed {
|
||||
* retention cannot be read, so they may need it indefinitely.
|
||||
*/
|
||||
offhostUntil: string | null;
|
||||
/** Database dumps on this server written under it. */
|
||||
localDumps: number;
|
||||
/** Off-host database dumps need it until this instant, or `"unknown"`. */
|
||||
offhostDumpsUntil: string | null;
|
||||
}
|
||||
|
||||
/** One key the kit lists, with its verdict. */
|
||||
@@ -204,9 +208,18 @@ export function sameKit(
|
||||
*/
|
||||
export function neededKeyIds(needs: BackupKeyNeeds, now: Date): Set<string> {
|
||||
const needed = new Set<string>();
|
||||
const stillThere = (until: string | null) =>
|
||||
until === null || Date.parse(until) > +now;
|
||||
for (const row of needs.stored) needed.add(row.keyId);
|
||||
for (const row of needs.offhost) {
|
||||
if (row.neededUntil === null || Date.parse(row.neededUntil) > +now) {
|
||||
if (stillThere(row.neededUntil)) needed.add(row.keyId);
|
||||
}
|
||||
// A database dump needs the key its rows were written under while it
|
||||
// exists: on this server until retention removes it, in the bucket until
|
||||
// the rule for `database/` expires it.
|
||||
for (const row of needs.dumps) {
|
||||
if (row.local > 0) needed.add(row.keyId);
|
||||
else if (row.offhost && stillThere(row.offhost.neededUntil)) {
|
||||
needed.add(row.keyId);
|
||||
}
|
||||
}
|
||||
@@ -229,6 +242,7 @@ export function applicationKitKeys(args: {
|
||||
return [...ids].sort().map((id) => {
|
||||
const fingerprint = args.fingerprintOf(id);
|
||||
const offhost = args.needs.offhost.find((row) => row.keyId === id);
|
||||
const dump = args.needs.dumps.find((row) => row.keyId === id);
|
||||
return {
|
||||
kind: "application" as const,
|
||||
id,
|
||||
@@ -241,6 +255,13 @@ export function applicationKitKeys(args: {
|
||||
args.needs.stored.find((row) => row.keyId === id)?.copies ?? 0,
|
||||
offhostUntil:
|
||||
offhost && needed.has(id) ? (offhost.neededUntil ?? "unknown") : null,
|
||||
localDumps: dump?.local ?? 0,
|
||||
offhostDumpsUntil:
|
||||
dump?.offhost &&
|
||||
(dump.offhost.neededUntil === null ||
|
||||
Date.parse(dump.offhost.neededUntil) > +args.now)
|
||||
? (dump.offhost.neededUntil ?? "unknown")
|
||||
: null,
|
||||
},
|
||||
problem: fingerprint === null ? ("not_configured" as const) : null,
|
||||
};
|
||||
@@ -248,7 +269,14 @@ export function applicationKitKeys(args: {
|
||||
}
|
||||
|
||||
function noNeed(active: boolean): KitKeyNeed {
|
||||
return { active, configured: true, storedCopies: 0, offhostUntil: null };
|
||||
return {
|
||||
active,
|
||||
configured: true,
|
||||
storedCopies: 0,
|
||||
offhostUntil: null,
|
||||
localDumps: 0,
|
||||
offhostDumpsUntil: null,
|
||||
};
|
||||
}
|
||||
|
||||
/** Pure: the off-host keys, from the two variables as the operator set them. */
|
||||
|
||||
@@ -124,3 +124,96 @@ describe("the keys stored backups still need", () => {
|
||||
expect(body.safeToDropRetiredKeys).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe("the keys the database dumps still need", () => {
|
||||
// Every row rotated and no stored copy on the old key: the view called the
|
||||
// old key safe to drop while the dumps from before the rotation, the
|
||||
// pre-migration one that is the way back from an update among them, could
|
||||
// only be restored with it.
|
||||
it("names the key a dump on this server or in the bucket was written under", async () => {
|
||||
const prisma = getPrismaClient();
|
||||
const admin = await prisma.user.create({
|
||||
data: { username: "dump-admin", email: "d@example.test", role: "ADMIN" },
|
||||
});
|
||||
const session = await prisma.session.create({
|
||||
data: { userId: admin.id, expiresAt: new Date(Date.now() + 60_000) },
|
||||
});
|
||||
cookieJar.set("healthlog_session", session.id);
|
||||
|
||||
const { mkdtempSync, writeFileSync, rmSync } = await import("node:fs");
|
||||
const { tmpdir } = await import("node:os");
|
||||
const { join } = await import("node:path");
|
||||
const { buildManifest, dumpBaseName } =
|
||||
await import("@/lib/database-backup/dump-core.mjs");
|
||||
const dir = mkdtempSync(join(tmpdir(), "hl-key-needs-"));
|
||||
const savedDir = process.env.DB_BACKUP_DIR;
|
||||
process.env.DB_BACKUP_DIR = dir;
|
||||
try {
|
||||
// A pre-migration dump taken while "old" was the active key.
|
||||
useKeys("old");
|
||||
const base = dumpBaseName(
|
||||
"pre-migration",
|
||||
new Date(Date.now() - 86_400_000),
|
||||
);
|
||||
writeFileSync(join(dir, `${base}.dump`), "PGDMP");
|
||||
writeFileSync(
|
||||
join(dir, `${base}.manifest.json`),
|
||||
JSON.stringify(
|
||||
buildManifest({
|
||||
kind: "pre-migration",
|
||||
file: `${base}.dump`,
|
||||
createdAt: new Date(),
|
||||
latestMigration: null,
|
||||
server: { version: "18", versionNum: 180000 },
|
||||
pgDump: null,
|
||||
archive: { bytes: 5, sha256: "x", entries: 1 },
|
||||
rowCounts: {},
|
||||
env: process.env,
|
||||
}),
|
||||
),
|
||||
);
|
||||
useKeys("cur");
|
||||
|
||||
const { GET } = await import("@/app/api/admin/encryption/status/route");
|
||||
const read = async () =>
|
||||
(await (await GET()).json()).data as {
|
||||
safeToDropRetiredKeys: boolean;
|
||||
backups: {
|
||||
dumps: Array<{ keyId: string; local: number }>;
|
||||
retiredKeysStillNeeded: string[];
|
||||
};
|
||||
};
|
||||
const local = await read();
|
||||
expect(local.backups.dumps).toEqual([
|
||||
expect.objectContaining({ keyId: "old", local: 1 }),
|
||||
]);
|
||||
expect(local.backups.retiredKeysStillNeeded).toEqual(["old"]);
|
||||
expect(local.safeToDropRetiredKeys).toBe(false);
|
||||
|
||||
// Gone from this server; its off-host copy, written under "old", is
|
||||
// still in a bucket whose rule for database/ nobody could read.
|
||||
rmSync(join(dir, `${base}.dump`));
|
||||
rmSync(join(dir, `${base}.manifest.json`));
|
||||
await prisma.databaseBackupRun.create({
|
||||
data: {
|
||||
startedAt: new Date(Date.now() - 86_400_000),
|
||||
finishedAt: new Date(Date.now() - 86_000_000),
|
||||
status: "succeeded",
|
||||
fileName: "healthlog-x-daily.dump",
|
||||
localResult: "not_persistent",
|
||||
offhostResult: "uploaded",
|
||||
activeKeyId: "old",
|
||||
},
|
||||
});
|
||||
const offhost = await read();
|
||||
expect(offhost.backups.dumps).toEqual([
|
||||
expect.objectContaining({ keyId: "old", local: 0 }),
|
||||
]);
|
||||
expect(offhost.backups.retiredKeysStillNeeded).toEqual(["old"]);
|
||||
} finally {
|
||||
process.env.DB_BACKUP_DIR = savedDir ?? "";
|
||||
if (savedDir === undefined) delete process.env.DB_BACKUP_DIR;
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -196,6 +196,9 @@ describe("the nightly database dump", () => {
|
||||
error: null,
|
||||
});
|
||||
expect(Number(runs[0].dumpBytes)).toBe(manifest.dump.bytes);
|
||||
// The key a restore of this dump needs, for the recovery kit.
|
||||
expect(runs[0].activeKeyId).toBe(manifest.keys.encryption.activeKeyId);
|
||||
expect(runs[0].activeKeyId).not.toBeNull();
|
||||
const health = await getDatabaseBackupHealth(
|
||||
new Date("2026-10-20T02:00:00Z"),
|
||||
backupEnv(),
|
||||
|
||||
Reference in new issue
Block a user