diff --git a/docs/ops/backup-restore.md b/docs/ops/backup-restore.md index 89a5aca58..6794929f2 100644 --- a/docs/ops/backup-restore.md +++ b/docs/ops/backup-restore.md @@ -146,8 +146,10 @@ once the lifecycle rule has expired every object written under it. A copy carries the database's ciphertext as stored, so it needs the application keys it was written under, also after a rotation. Keep a retired `ENCRYPTION_KEYS` entry until no copy you may still restore needs it. **Admin → -Encryption → Keys the backups still need** lists them. Off-host objects also -need `BACKUP_ENCRYPTION_KEY` (or the previous key that wrote them). +Encryption → Keys the backups still need** lists them, the database dumps +included: a dump needs the key that was active when it was taken, for as long +as it is kept on this server or in the bucket. Off-host objects also need +`BACKUP_ENCRYPTION_KEY` (or the previous key that wrote them). ## Size and memory diff --git a/docs/ops/encryption-key-rotation.md b/docs/ops/encryption-key-rotation.md index a1417998b..38f05c070 100644 --- a/docs/ops/encryption-key-rotation.md +++ b/docs/ops/encryption-key-rotation.md @@ -137,10 +137,15 @@ under `v1` (the synthetic id assigned to the existing `ENCRYPTION_KEY`). > So before step 4, also check **Admin → Encryption → Keys the backups still > need**. It lists, per key id, the stored copies that need it with the oldest > date, and when the last off-host copy needing it expires under the bucket's -> lifecycle rule. Keep the old key in `ENCRYPTION_KEYS` until it is no longer -> listed there (the weekly copy replaces itself within a week; delete old -> uploaded copies you no longer want, and wait out the off-host retention), or -> accept that those copies cannot be restored. Copies written before v1.39.3 +> lifecycle rule. The database dumps are listed too: a dump is restored only +> with the key that was active when it was taken (`db-restore` refuses +> without it), so the old key stays listed while a nightly or pre-migration +> dump taken under it is on this server, and until the bucket's rule for +> `database/` has expired the off-host copies. Keep the old key in +> `ENCRYPTION_KEYS` until it is no longer listed there (the weekly copy +> replaces itself within a week; delete old uploaded copies you no longer +> want, and wait out the dump and off-host retention), or accept that those +> copies cannot be restored. Copies written before v1.39.3 > did not record their keys and are listed as such: treat them as needing > every key that existed when they were written. > diff --git a/messages/de.json b/messages/de.json index add8370ee..c6e47f7d8 100644 --- a/messages/de.json +++ b/messages/de.json @@ -8418,7 +8418,10 @@ "columnsHide": "Tabelle ausblenden", "columnsShowAll": "Alle {count} Spalten anzeigen", "columnsShowWithRows": "Nur Spalten mit Zeilen", - "columnsNoneWithRows": "Noch keine verschlüsselte Spalte enthält Zeilen." + "columnsNoneWithRows": "Noch keine verschlüsselte Spalte enthält Zeilen.", + "backupsDumpLocalLine": "Schlüssel {key}: {count} Datenbank-Dumps auf diesem Server wurden unter ihm geschrieben; er wird gebraucht, bis die Aufbewahrung sie entfernt.", + "backupsDumpOffhostLine": "Schlüssel {key}: Der neueste Datenbank-Dump, der unter ihm geschrieben wurde, kam am {when} in den Bucket; dort gebraucht bis {until}", + "backupsDumpOffhostLineUnknown": "Schlüssel {key}: Der neueste Datenbank-Dump, der unter ihm geschrieben wurde, kam am {when} in den Bucket. Wie lange der Bucket database/ behält, ist unbekannt; behalte den Schlüssel, bis dieser Dump sicher weg ist." }, "maintenance": { "title": "Pflege der Messwert-Tabelle", @@ -8604,7 +8607,10 @@ "offhostUnknown": "Wird von externen Kopien gebraucht, solange der Bucket sie aufbewahrt.", "backupActive": "Versiegelt jede neue externe Kopie.", "backupPrevious": "Öffnet externe Kopien, die vor dem letzten Schlüsselwechsel geschrieben wurden.", - "token": "Signiert Sitzungen, Tokens und Freigabelinks. Ohne ihn werden alle abgemeldet, Daten gehen aber nicht verloren." + "token": "Signiert Sitzungen, Tokens und Freigabelinks. Ohne ihn werden alle abgemeldet, Daten gehen aber nicht verloren.", + "localDumps": "Wird von {count} Datenbank-Dumps auf diesem Server gebraucht.", + "offhostDumpsUntil": "Wird von externen Datenbank-Dumps bis {date} gebraucht.", + "offhostDumpsUnknown": "Wird von externen Datenbank-Dumps gebraucht, solange der Bucket sie aufbewahrt." }, "source": "Gesetzt in {variable}.", "status": { diff --git a/messages/en.json b/messages/en.json index 0e65dbb05..54cb92bff 100644 --- a/messages/en.json +++ b/messages/en.json @@ -8418,7 +8418,10 @@ "columnsHide": "Hide the table", "columnsShowAll": "Show all {count} columns", "columnsShowWithRows": "Only columns with rows", - "columnsNoneWithRows": "No encrypted column holds rows yet." + "columnsNoneWithRows": "No encrypted column holds rows yet.", + "backupsDumpLocalLine": "Key {key}: {count} database dumps on this server were written under it; it is needed until retention removes them.", + "backupsDumpOffhostLine": "Key {key}: the newest database dump written under it went into the bucket on {when}; needed there until {until}", + "backupsDumpOffhostLineUnknown": "Key {key}: the newest database dump written under it went into the bucket on {when}. How long the bucket keeps database/ is unknown, so keep the key until you know that dump is gone." }, "maintenance": { "title": "Measurement table maintenance", @@ -8604,7 +8607,10 @@ "offhostUnknown": "Needed by off-host copies for as long as the bucket keeps them.", "backupActive": "Seals every new off-host copy.", "backupPrevious": "Opens off-host copies written before the last key change.", - "token": "Signs sessions, tokens and share links. Without it everyone is signed out, but no data is lost." + "token": "Signs sessions, tokens and share links. Without it everyone is signed out, but no data is lost.", + "localDumps": "Needed by {count} database dumps on this server.", + "offhostDumpsUntil": "Needed by off-host database dumps until {date}.", + "offhostDumpsUnknown": "Needed by off-host database dumps for as long as the bucket keeps them." }, "source": "Set in {variable}.", "status": { diff --git a/messages/es.json b/messages/es.json index 0fd0bf0e2..84df9344c 100644 --- a/messages/es.json +++ b/messages/es.json @@ -8418,7 +8418,10 @@ "columnsHide": "Ocultar la tabla", "columnsShowAll": "Mostrar las {count} columnas", "columnsShowWithRows": "Solo columnas con filas", - "columnsNoneWithRows": "Ninguna columna cifrada contiene filas todavía." + "columnsNoneWithRows": "Ninguna columna cifrada contiene filas todavía.", + "backupsDumpLocalLine": "Clave {key}: {count} volcados de la base de datos en este servidor se escribieron con ella; se necesita hasta que la retención los elimine.", + "backupsDumpOffhostLine": "Clave {key}: el volcado de la base de datos más reciente escrito con ella entró en el bucket el {when}; se necesita allí hasta el {until}", + "backupsDumpOffhostLineUnknown": "Clave {key}: el volcado de la base de datos más reciente escrito con ella entró en el bucket el {when}. No se sabe cuánto tiempo conserva el bucket database/, así que conserva la clave hasta saber que ese volcado ya no está." }, "maintenance": { "title": "Mantenimiento de la tabla de mediciones", @@ -8604,7 +8607,10 @@ "offhostUnknown": "La necesitan las copias fuera del host mientras el bucket las conserve.", "backupActive": "Sella cada nueva copia fuera del host.", "backupPrevious": "Abre las copias fuera del host escritas antes del último cambio de clave.", - "token": "Firma sesiones, tokens y enlaces para compartir. Sin ella se cierra la sesión de todo el mundo, pero no se pierden datos." + "token": "Firma sesiones, tokens y enlaces para compartir. Sin ella se cierra la sesión de todo el mundo, pero no se pierden datos.", + "localDumps": "La necesitan {count} volcados de la base de datos en este servidor.", + "offhostDumpsUntil": "La necesitan los volcados de la base de datos fuera del host hasta el {date}.", + "offhostDumpsUnknown": "La necesitan los volcados de la base de datos fuera del host mientras el bucket los conserve." }, "source": "Definida en {variable}.", "status": { diff --git a/messages/fr.json b/messages/fr.json index bfd69acd3..2568f52f8 100644 --- a/messages/fr.json +++ b/messages/fr.json @@ -8418,7 +8418,10 @@ "columnsHide": "Masquer le tableau", "columnsShowAll": "Afficher les {count} colonnes", "columnsShowWithRows": "Seulement les colonnes avec des lignes", - "columnsNoneWithRows": "Aucune colonne chiffrée ne contient encore de lignes." + "columnsNoneWithRows": "Aucune colonne chiffrée ne contient encore de lignes.", + "backupsDumpLocalLine": "Clé {key} : {count} dumps de la base de données sur ce serveur ont été écrits avec elle ; elle est nécessaire jusqu'à ce que la rétention les supprime.", + "backupsDumpOffhostLine": "Clé {key} : le dump de la base de données le plus récent écrit avec elle est arrivé dans le bucket le {when} ; nécessaire jusqu'au {until}", + "backupsDumpOffhostLineUnknown": "Clé {key} : le dump de la base de données le plus récent écrit avec elle est arrivé dans le bucket le {when}. La durée de conservation de database/ dans le bucket est inconnue : gardez la clé jusqu'à être sûr que ce dump a disparu." }, "maintenance": { "title": "Maintenance de la table des mesures", @@ -8604,7 +8607,10 @@ "offhostUnknown": "Nécessaire aux copies hors hôte tant que le bucket les conserve.", "backupActive": "Scelle chaque nouvelle copie hors hôte.", "backupPrevious": "Ouvre les copies hors hôte écrites avant le dernier changement de clé.", - "token": "Signe les sessions, les jetons et les liens de partage. Sans elle, tout le monde est déconnecté, mais aucune donnée n'est perdue." + "token": "Signe les sessions, les jetons et les liens de partage. Sans elle, tout le monde est déconnecté, mais aucune donnée n'est perdue.", + "localDumps": "Nécessaire à {count} dumps de la base de données sur ce serveur.", + "offhostDumpsUntil": "Nécessaire aux dumps de la base de données hors hôte jusqu'au {date}.", + "offhostDumpsUnknown": "Nécessaire aux dumps de la base de données hors hôte tant que le bucket les conserve." }, "source": "Définie dans {variable}.", "status": { diff --git a/messages/it.json b/messages/it.json index 6f6e92dce..cf5285ada 100644 --- a/messages/it.json +++ b/messages/it.json @@ -8418,7 +8418,10 @@ "columnsHide": "Nascondi la tabella", "columnsShowAll": "Mostra tutte le {count} colonne", "columnsShowWithRows": "Solo colonne con righe", - "columnsNoneWithRows": "Nessuna colonna cifrata contiene ancora righe." + "columnsNoneWithRows": "Nessuna colonna cifrata contiene ancora righe.", + "backupsDumpLocalLine": "Chiave {key}: {count} dump del database su questo server sono stati scritti con essa; serve finché la conservazione non li rimuove.", + "backupsDumpOffhostLine": "Chiave {key}: il dump del database più recente scritto con essa è entrato nel bucket il {when}; serve lì fino al {until}", + "backupsDumpOffhostLineUnknown": "Chiave {key}: il dump del database più recente scritto con essa è entrato nel bucket il {when}. Non si sa per quanto il bucket conservi database/, quindi conserva la chiave finché non sai che quel dump non c'è più." }, "maintenance": { "title": "Manutenzione della tabella delle misurazioni", @@ -8604,7 +8607,10 @@ "offhostUnknown": "Serve alle copie fuori host finché il bucket le conserva.", "backupActive": "Sigilla ogni nuova copia fuori host.", "backupPrevious": "Apre le copie fuori host scritte prima dell'ultimo cambio di chiave.", - "token": "Firma sessioni, token e link di condivisione. Senza di essa tutti vengono disconnessi, ma nessun dato va perso." + "token": "Firma sessioni, token e link di condivisione. Senza di essa tutti vengono disconnessi, ma nessun dato va perso.", + "localDumps": "Serve a {count} dump del database su questo server.", + "offhostDumpsUntil": "Serve ai dump del database fuori host fino al {date}.", + "offhostDumpsUnknown": "Serve ai dump del database fuori host finché il bucket li conserva." }, "source": "Impostata in {variable}.", "status": { diff --git a/messages/ko.json b/messages/ko.json index e5932cb29..eb473420e 100644 --- a/messages/ko.json +++ b/messages/ko.json @@ -8393,7 +8393,10 @@ "columnsHide": "표 숨기기", "columnsShowAll": "열 {count}개 모두 보기", "columnsShowWithRows": "행이 있는 열만", - "columnsNoneWithRows": "아직 행이 있는 암호화 열이 없어요." + "columnsNoneWithRows": "아직 행이 있는 암호화 열이 없어요.", + "backupsDumpLocalLine": "키 {key}: 이 서버의 데이터베이스 덤프 {count}개가 이 키로 작성되었습니다. 보존 정책이 덤프를 지울 때까지 필요합니다.", + "backupsDumpOffhostLine": "키 {key}: 이 키로 작성된 최신 데이터베이스 덤프가 {when}에 버킷에 저장되었습니다. {until}까지 필요합니다", + "backupsDumpOffhostLineUnknown": "키 {key}: 이 키로 작성된 최신 데이터베이스 덤프가 {when}에 버킷에 저장되었습니다. 버킷이 database/를 얼마나 보관하는지 알 수 없으므로 그 덤프가 사라진 것을 확인할 때까지 키를 유지하세요." }, "maintenance": { "title": "측정값 테이블 유지 관리", @@ -8604,7 +8607,10 @@ "offhostUnknown": "버킷이 외부 보관 사본을 보관하는 동안 필요합니다.", "backupActive": "새 외부 보관 사본을 모두 봉인합니다.", "backupPrevious": "마지막 키 변경 전에 기록된 외부 보관 사본을 엽니다.", - "token": "세션, 토큰, 공유 링크에 서명합니다. 이 키가 없으면 모두 로그아웃되지만 데이터는 잃지 않습니다." + "token": "세션, 토큰, 공유 링크에 서명합니다. 이 키가 없으면 모두 로그아웃되지만 데이터는 잃지 않습니다.", + "localDumps": "이 서버의 데이터베이스 덤프 {count}개에 필요합니다.", + "offhostDumpsUntil": "{date}까지 외부 보관 데이터베이스 덤프에 필요합니다.", + "offhostDumpsUnknown": "버킷이 외부 보관 데이터베이스 덤프를 보관하는 동안 필요합니다." }, "source": "{variable}에 설정됨.", "status": { diff --git a/messages/nl.json b/messages/nl.json index a1bc952a7..953d83e49 100644 --- a/messages/nl.json +++ b/messages/nl.json @@ -8418,7 +8418,10 @@ "columnsHide": "Tabel verbergen", "columnsShowAll": "Alle {count} kolommen tonen", "columnsShowWithRows": "Alleen kolommen met rijen", - "columnsNoneWithRows": "Nog geen enkele versleutelde kolom bevat rijen." + "columnsNoneWithRows": "Nog geen enkele versleutelde kolom bevat rijen.", + "backupsDumpLocalLine": "Sleutel {key}: {count} databasedumps op deze server zijn onder deze sleutel geschreven; hij is nodig tot de bewaartermijn ze verwijdert.", + "backupsDumpOffhostLine": "Sleutel {key}: de nieuwste databasedump onder deze sleutel is op {when} in de bucket gezet; daar nodig tot {until}", + "backupsDumpOffhostLineUnknown": "Sleutel {key}: de nieuwste databasedump onder deze sleutel is op {when} in de bucket gezet. Hoe lang de bucket database/ bewaart, is onbekend, dus bewaar de sleutel tot je zeker weet dat die dump weg is." }, "maintenance": { "title": "Onderhoud van de metingentabel", @@ -8604,7 +8607,10 @@ "offhostUnknown": "Nodig voor externe kopieën zolang de bucket ze bewaart.", "backupActive": "Verzegelt elke nieuwe externe kopie.", "backupPrevious": "Opent externe kopieën die vóór de laatste sleutelwissel zijn geschreven.", - "token": "Ondertekent sessies, tokens en deellinks. Zonder deze sleutel wordt iedereen afgemeld, maar gaan er geen gegevens verloren." + "token": "Ondertekent sessies, tokens en deellinks. Zonder deze sleutel wordt iedereen afgemeld, maar gaan er geen gegevens verloren.", + "localDumps": "Nodig voor {count} databasedumps op deze server.", + "offhostDumpsUntil": "Nodig voor externe databasedumps tot {date}.", + "offhostDumpsUnknown": "Nodig voor externe databasedumps zolang de bucket ze bewaart." }, "source": "Ingesteld in {variable}.", "status": { diff --git a/messages/pl.json b/messages/pl.json index b2e166081..3dd9754b5 100644 --- a/messages/pl.json +++ b/messages/pl.json @@ -8418,7 +8418,10 @@ "columnsHide": "Ukryj tabelę", "columnsShowAll": "Pokaż wszystkie kolumny ({count})", "columnsShowWithRows": "Tylko kolumny z wierszami", - "columnsNoneWithRows": "Żadna zaszyfrowana kolumna nie zawiera jeszcze wierszy." + "columnsNoneWithRows": "Żadna zaszyfrowana kolumna nie zawiera jeszcze wierszy.", + "backupsDumpLocalLine": "Klucz {key}: {count} zrzutów bazy danych na tym serwerze zapisano pod nim; jest potrzebny, dopóki retencja ich nie usunie.", + "backupsDumpOffhostLine": "Klucz {key}: najnowszy zrzut bazy danych zapisany pod nim trafił do bucketu {when}; potrzebny tam do {until}", + "backupsDumpOffhostLineUnknown": "Klucz {key}: najnowszy zrzut bazy danych zapisany pod nim trafił do bucketu {when}. Nie wiadomo, jak długo bucket przechowuje database/, więc zachowaj klucz, dopóki nie masz pewności, że tego zrzutu już nie ma." }, "maintenance": { "title": "Konserwacja tabeli pomiarów", @@ -8604,7 +8607,10 @@ "offhostUnknown": "Potrzebny kopiom poza hostem, dopóki bucket je przechowuje.", "backupActive": "Pieczętuje każdą nową kopię poza hostem.", "backupPrevious": "Otwiera kopie poza hostem zapisane przed ostatnią zmianą klucza.", - "token": "Podpisuje sesje, tokeny i linki udostępniania. Bez niego wszyscy zostaną wylogowani, ale żadne dane nie przepadną." + "token": "Podpisuje sesje, tokeny i linki udostępniania. Bez niego wszyscy zostaną wylogowani, ale żadne dane nie przepadną.", + "localDumps": "Potrzebny zrzutom bazy danych na tym serwerze: {count}.", + "offhostDumpsUntil": "Potrzebny zrzutom bazy danych poza hostem do {date}.", + "offhostDumpsUnknown": "Potrzebny zrzutom bazy danych poza hostem, dopóki bucket je przechowuje." }, "source": "Ustawiony w {variable}.", "status": { diff --git a/messages/pt-BR.json b/messages/pt-BR.json index 78a7998f3..9c6006efd 100644 --- a/messages/pt-BR.json +++ b/messages/pt-BR.json @@ -8418,7 +8418,10 @@ "columnsHide": "Ocultar a tabela", "columnsShowAll": "Mostrar todas as {count} colunas", "columnsShowWithRows": "Somente colunas com linhas", - "columnsNoneWithRows": "Nenhuma coluna criptografada contém linhas ainda." + "columnsNoneWithRows": "Nenhuma coluna criptografada contém linhas ainda.", + "backupsDumpLocalLine": "Chave {key}: {count} dumps do banco de dados neste servidor foram gravados com ela; ela é necessária até que a retenção os remova.", + "backupsDumpOffhostLine": "Chave {key}: o dump do banco de dados mais recente gravado com ela foi para o bucket em {when}; necessária lá até {until}", + "backupsDumpOffhostLineUnknown": "Chave {key}: o dump do banco de dados mais recente gravado com ela foi para o bucket em {when}. Não se sabe por quanto tempo o bucket mantém database/, então mantenha a chave até saber que aquele dump se foi." }, "maintenance": { "title": "Manutenção da tabela de medições", @@ -8604,7 +8607,10 @@ "offhostUnknown": "Necessária para as cópias fora do host enquanto o bucket as mantiver.", "backupActive": "Sela cada nova cópia fora do host.", "backupPrevious": "Abre as cópias fora do host gravadas antes da última troca de chave.", - "token": "Assina sessões, tokens e links de compartilhamento. Sem ela, todos são desconectados, mas nenhum dado é perdido." + "token": "Assina sessões, tokens e links de compartilhamento. Sem ela, todos são desconectados, mas nenhum dado é perdido.", + "localDumps": "Necessária para {count} dumps do banco de dados neste servidor.", + "offhostDumpsUntil": "Necessária para os dumps do banco de dados fora do host até {date}.", + "offhostDumpsUnknown": "Necessária para os dumps do banco de dados fora do host enquanto o bucket os mantiver." }, "source": "Definida em {variable}.", "status": { diff --git a/prisma/migrations/0489_database_backup_run_key/migration.sql b/prisma/migrations/0489_database_backup_run_key/migration.sql new file mode 100644 index 000000000..3160bc014 --- /dev/null +++ b/prisma/migrations/0489_database_backup_run_key/migration.sql @@ -0,0 +1,18 @@ +-- The nightly database dump records the application key its rows were +-- written under (the manifest's active key id). A dump is restored only with +-- that key, so the recovery kit and "keys the backups still need" list it for +-- as long as the off-host copy of the dump may still be in the bucket. +-- +-- Nullable, no default, no backfill: a run recorded before this column reads +-- as "key not recorded", and its dump ages out of the bucket like any other. +-- Reversibility: dropping the column restores the previous schema. +-- +-- @retry-on-lock-timeout + +DO $migration$ +BEGIN + PERFORM set_config('lock_timeout', '5s', true); + ALTER TABLE "database_backup_runs" + ADD COLUMN IF NOT EXISTS "active_key_id" TEXT; +END +$migration$; diff --git a/prisma/schema.prisma b/prisma/schema.prisma index 78fd0520b..648b62a75 100644 --- a/prisma/schema.prisma +++ b/prisma/schema.prisma @@ -6032,6 +6032,11 @@ model DatabaseBackupRun { offhostResult String @map("offhost_result") /// Why the run failed or was skipped, trimmed. Never key material. error String? + /// The application key id the dump's rows were written under (its + /// manifest's active key). A restore refuses without it. Reader: + /// `readBackupKeyNeeds`, which keeps the key in the recovery kit while the + /// off-host copy of this dump may still be in the bucket. + activeKeyId String? @map("active_key_id") @@index([startedAt]) @@map("database_backup_runs") diff --git a/scripts/seed-demo.ts b/scripts/seed-demo.ts index 6974d57c3..c5a37c7c6 100644 --- a/scripts/seed-demo.ts +++ b/scripts/seed-demo.ts @@ -3975,6 +3975,7 @@ export async function seed() { stored: [], unrecorded: { copies: 0, oldestAt: null }, offhost: [], + dumps: [], offhostExpirationDays: null, retiredKeysStillNeeded: [], }, diff --git a/src/components/admin/__tests__/encryption-legacy-badge.test.tsx b/src/components/admin/__tests__/encryption-legacy-badge.test.tsx index 2effd2e35..793abbba9 100644 --- a/src/components/admin/__tests__/encryption-legacy-badge.test.tsx +++ b/src/components/admin/__tests__/encryption-legacy-badge.test.tsx @@ -44,6 +44,7 @@ function statusWith(configuredKeyCount: number) { stored: [], unrecorded: { copies: 0, oldestAt: null }, offhost: [], + dumps: [], offhostExpirationDays: null, retiredKeysStillNeeded: [], }, diff --git a/src/components/admin/__tests__/recovery-kit-card.test.tsx b/src/components/admin/__tests__/recovery-kit-card.test.tsx index 83722b8cb..35bbb71f8 100644 --- a/src/components/admin/__tests__/recovery-kit-card.test.tsx +++ b/src/components/admin/__tests__/recovery-kit-card.test.tsx @@ -40,6 +40,8 @@ const need = (over: Partial = {}): KitKey["need"] => ({ configured: true, storedCopies: 0, offhostUntil: null, + localDumps: 0, + offhostDumpsUntil: null, ...over, }); diff --git a/src/components/admin/encryption-section.tsx b/src/components/admin/encryption-section.tsx index aa4441c08..ebf362cb6 100644 --- a/src/components/admin/encryption-section.tsx +++ b/src/components/admin/encryption-section.tsx @@ -88,6 +88,11 @@ interface EncryptionStatus { lastWrittenAt: string; neededUntil: string | null; }>; + dumps: Array<{ + keyId: string; + local: number; + offhost: { lastWrittenAt: string; neededUntil: string | null } | null; + }>; offhostExpirationDays: number | null; retiredKeysStillNeeded: string[]; }; @@ -306,9 +311,39 @@ export function EncryptionSection() { })} ))} + {s.backups.dumps.map((row) => ( +
  • + {[ + row.local > 0 + ? t("admin.section.encryption.backupsDumpLocalLine", { + key: row.keyId, + count: row.local, + }) + : null, + row.offhost + ? row.offhost.neededUntil + ? t("admin.section.encryption.backupsDumpOffhostLine", { + key: row.keyId, + when: fmt.dateTime(row.offhost.lastWrittenAt), + until: fmt.dateTime(row.offhost.neededUntil), + }) + : t( + "admin.section.encryption.backupsDumpOffhostLineUnknown", + { + key: row.keyId, + when: fmt.dateTime(row.offhost.lastWrittenAt), + }, + ) + : null, + ] + .filter((line): line is string => line !== null) + .join(" ")} +
    • + ))} {s.backups.stored.length === 0 && s.backups.unrecorded.copies === 0 && - s.backups.offhost.length === 0 ? ( + s.backups.offhost.length === 0 && + s.backups.dumps.length === 0 ? (
  • {t("admin.section.encryption.backupsNone")}
    • diff --git a/src/components/admin/recovery-kit-card.tsx b/src/components/admin/recovery-kit-card.tsx index b08a9282f..915754093 100644 --- a/src/components/admin/recovery-kit-card.tsx +++ b/src/components/admin/recovery-kit-card.tsx @@ -138,6 +138,20 @@ function KeyNeeds({ k }: { k: KitKey }) { }), ); } + if (k.need.localDumps > 0) { + lines.push( + t("admin.recoveryKit.need.localDumps", { count: k.need.localDumps }), + ); + } + if (k.need.offhostDumpsUntil === "unknown") { + lines.push(t("admin.recoveryKit.need.offhostDumpsUnknown")); + } else if (k.need.offhostDumpsUntil) { + lines.push( + t("admin.recoveryKit.need.offhostDumpsUntil", { + date: fmt.date(new Date(k.need.offhostDumpsUntil)), + }), + ); + } } lines.push(t("admin.recoveryKit.source", { variable: k.source })); return

    {lines.join(" ")}

    ; diff --git a/src/lib/database-backup/run.ts b/src/lib/database-backup/run.ts index 0524f5093..fd76ebe25 100644 --- a/src/lib/database-backup/run.ts +++ b/src/lib/database-backup/run.ts @@ -156,6 +156,7 @@ export async function runDatabaseBackup( }); let scratch: string | null = null; + let activeKeyId: string | null = null; try { if (!databaseBackupEnabled(env)) { report.status = "skipped"; @@ -182,6 +183,7 @@ export async function runDatabaseBackup( deadline: options.deadline, }); if (keepLocal) report.local = "written"; + activeKeyId = dump.manifest.keys.encryption.activeKeyId; report.fileName = basename(dump.dumpPath); report.dumpBytes = dump.manifest.dump.bytes; @@ -233,6 +235,7 @@ export async function runDatabaseBackup( localResult: report.local, offhostResult: report.offhost, error: report.error, + activeKeyId, }, }); await prisma.databaseBackupRun.deleteMany({ diff --git a/src/lib/recovery/__tests__/kit-rules.test.ts b/src/lib/recovery/__tests__/kit-rules.test.ts index f05e4b624..19636e3c3 100644 --- a/src/lib/recovery/__tests__/kit-rules.test.ts +++ b/src/lib/recovery/__tests__/kit-rules.test.ts @@ -31,6 +31,7 @@ const NO_NEEDS: BackupKeyNeeds = { stored: [], unrecorded: { copies: 0, oldestAt: null }, offhost: [], + dumps: [], offhostExpirationDays: null, retiredKeysStillNeeded: [], }; @@ -76,6 +77,43 @@ describe("which keys the kit lists", () => { }); }); + // A dump is restored only with the key its rows were written under; with + // it out of the kit, a rotation read as finished while every dump from + // before it, the pre-migration ones among them, still needed the old key. + it("keeps a retired key while a database dump written under it exists", () => { + const local: BackupKeyNeeds = { + ...NO_NEEDS, + dumps: [{ keyId: "v1", local: 2, offhost: null }], + }; + expect(neededKeyIds(local, NOW)).toEqual(new Set(["v1"])); + const [v1] = appKeys(["v2"], "v2", local); + expect(v1).toMatchObject({ + id: "v1", + problem: "not_configured", + need: { localDumps: 2, offhostDumpsUntil: null }, + }); + + const offhost = (neededUntil: string | null): BackupKeyNeeds => ({ + ...NO_NEEDS, + dumps: [ + { + keyId: "v1", + local: 0, + offhost: { lastWrittenAt: NOW.toISOString(), neededUntil }, + }, + ], + }); + const later = new Date(NOW.getTime() + 3 * DAY).toISOString(); + expect(neededKeyIds(offhost(later), NOW).has("v1")).toBe(true); + expect(appKeys(["v1", "v2"], "v2", offhost(later))[0].need).toMatchObject({ + offhostDumpsUntil: later, + }); + // Unknown retention keeps it; an expiry that has passed lets it go. + expect(neededKeyIds(offhost(null), NOW).has("v1")).toBe(true); + const earlier = new Date(NOW.getTime() - DAY).toISOString(); + expect(neededKeyIds(offhost(earlier), NOW).has("v1")).toBe(false); + }); + it("keeps an off-host key while objects inside the retention need it", () => { const row = (neededUntil: string | null) => ({ ...NO_NEEDS, diff --git a/src/lib/recovery/backup-key-needs.ts b/src/lib/recovery/backup-key-needs.ts index fa2940577..cc83f4e05 100644 --- a/src/lib/recovery/backup-key-needs.ts +++ b/src/lib/recovery/backup-key-needs.ts @@ -10,10 +10,25 @@ * bucket the last night an object needing it went in, plus how long the * bucket's lifecycle rule keeps objects after that. * + * The database dumps count too. A dump is restored only with the key its rows + * were written under (its manifest's active key; `db-restore` refuses without + * it), so that key is needed for as long as the dump exists: a dump kept on + * this server until retention removes it, an off-host one until the bucket's + * rule for `database/` expires it. Without them, a rotation read as safe to + * finish a week later while every dump from before it, the pre-migration + * dumps that are the way back from an update among them, still needed the + * retired key. + * * Two readers ask it: the admin encryption view, which shows the table, and * the recovery kit, which lists every key a restore needs. One definition, so * the kit cannot drop a key the encryption view still calls needed. */ +import { + isGoodSet, + listDumpSets, + readManifestFile, + resolveBackupDir, +} from "@/lib/database-backup/dump-core.mjs"; import { prisma } from "@/lib/db"; import { offhostBackupConfigured, @@ -32,50 +47,109 @@ export interface BackupKeyNeeds { /** When the last object needing this key expires; null when unknown. */ neededUntil: string | null; }>; + /** + * Per key id, the database dumps written under it: how many are kept on + * this server, and the newest one that went off-host with when the bucket + * expires it (`neededUntil` null when that cannot be read). + */ + dumps: Array<{ + keyId: string; + local: number; + offhost: { lastWrittenAt: string; neededUntil: string | null } | null; + }>; /** The bucket's expiry, when it has one the credential may read. */ offhostExpirationDays: number | null; /** Retired key ids some backup still needs (never the active one). */ retiredKeysStillNeeded: string[]; } +/** + * The active key of every good dump in `dir`, one entry per dump. A manifest + * that cannot be read names no key; `db-restore` refuses such a dump anyway. + */ +export async function localDumpKeyIds(dir: string): Promise { + const keys: string[] = []; + for (const set of listDumpSets(dir).filter(isGoodSet)) { + try { + const manifest = await readManifestFile(set.manifest as string); + const id = manifest.keys.encryption.activeKeyId; + if (id) keys.push(id); + } catch { + // Unreadable: no key to name. + } + } + return keys; +} + export async function readBackupKeyNeeds( activeKeyId: string, now: Date, + options: { dumpDir?: string } = {}, ): Promise { - const [stored, unrecorded, offhostRows] = await Promise.all([ - prisma.$queryRaw< - Array<{ key_id: string; copies: number; oldest_at: Date }> - >` + const [stored, unrecorded, offhostRows, dumpUploads, localDumpKeys] = + await Promise.all([ + prisma.$queryRaw< + Array<{ key_id: string; copies: number; oldest_at: Date }> + >` SELECT k AS key_id, count(*)::int AS copies, min(created_at) AS oldest_at FROM data_backups, unnest(inner_key_ids) AS k WHERE inner_key_ids_recorded GROUP BY k ORDER BY k `, - prisma.dataBackup.aggregate({ - where: { innerKeyIdsRecorded: false }, - _count: { _all: true }, - _min: { createdAt: true }, - }), - prisma.offhostBackupKeyUse.findMany({ orderBy: { keyId: "asc" } }), - ]); + prisma.dataBackup.aggregate({ + where: { innerKeyIdsRecorded: false }, + _count: { _all: true }, + _min: { createdAt: true }, + }), + prisma.offhostBackupKeyUse.findMany({ orderBy: { keyId: "asc" } }), + prisma.databaseBackupRun.groupBy({ + by: ["activeKeyId"], + where: { offhostResult: "uploaded", activeKeyId: { not: null } }, + _max: { startedAt: true }, + }), + localDumpKeyIds(options.dumpDir ?? resolveBackupDir()), + ]); const lifecycle = - offhostRows.length > 0 && offhostBackupConfigured() + (offhostRows.length > 0 || dumpUploads.length > 0) && + offhostBackupConfigured() ? await probeOffhostLifecycle() : null; const expirationDays = lifecycle?.state === "configured" ? lifecycle.expirationDays : null; + const dumpExpirationDays = + lifecycle?.database.state === "configured" + ? lifecycle.database.expirationDays + : null; + const expiresAfter = (writtenAt: Date, days: number | null) => + days === null + ? null + : new Date(writtenAt.getTime() + (days + 1) * 86_400_000).toISOString(); + + const dumpKeyIds = new Set([ + ...localDumpKeys, + ...dumpUploads.map((row) => row.activeKeyId as string), + ]); + const dumps = [...dumpKeyIds].sort().map((keyId) => { + const upload = dumpUploads.find((row) => row.activeKeyId === keyId); + const writtenAt = upload?._max.startedAt ?? null; + return { + keyId, + local: localDumpKeys.filter((id) => id === keyId).length, + offhost: writtenAt + ? { + lastWrittenAt: writtenAt.toISOString(), + neededUntil: expiresAfter(writtenAt, dumpExpirationDays), + } + : null, + }; + }); const offhost = offhostRows.map((row) => ({ keyId: row.keyId, firstWrittenAt: row.firstWrittenAt.toISOString(), lastWrittenAt: row.lastWrittenAt.toISOString(), - neededUntil: - expirationDays === null - ? null - : new Date( - row.lastWrittenAt.getTime() + (expirationDays + 1) * 86_400_000, - ).toISOString(), + neededUntil: expiresAfter(row.lastWrittenAt, expirationDays), })); const needs: BackupKeyNeeds = { @@ -89,6 +163,7 @@ export async function readBackupKeyNeeds( oldestAt: unrecorded._min.createdAt?.toISOString() ?? null, }, offhost, + dumps, offhostExpirationDays: expirationDays, retiredKeysStillNeeded: [], }; diff --git a/src/lib/recovery/kit-rules.ts b/src/lib/recovery/kit-rules.ts index e7edfd791..a59b50882 100644 --- a/src/lib/recovery/kit-rules.ts +++ b/src/lib/recovery/kit-rules.ts @@ -82,6 +82,10 @@ export interface KitKeyNeed { * retention cannot be read, so they may need it indefinitely. */ offhostUntil: string | null; + /** Database dumps on this server written under it. */ + localDumps: number; + /** Off-host database dumps need it until this instant, or `"unknown"`. */ + offhostDumpsUntil: string | null; } /** One key the kit lists, with its verdict. */ @@ -204,9 +208,18 @@ export function sameKit( */ export function neededKeyIds(needs: BackupKeyNeeds, now: Date): Set { const needed = new Set(); + const stillThere = (until: string | null) => + until === null || Date.parse(until) > +now; for (const row of needs.stored) needed.add(row.keyId); for (const row of needs.offhost) { - if (row.neededUntil === null || Date.parse(row.neededUntil) > +now) { + if (stillThere(row.neededUntil)) needed.add(row.keyId); + } + // A database dump needs the key its rows were written under while it + // exists: on this server until retention removes it, in the bucket until + // the rule for `database/` expires it. + for (const row of needs.dumps) { + if (row.local > 0) needed.add(row.keyId); + else if (row.offhost && stillThere(row.offhost.neededUntil)) { needed.add(row.keyId); } } @@ -229,6 +242,7 @@ export function applicationKitKeys(args: { return [...ids].sort().map((id) => { const fingerprint = args.fingerprintOf(id); const offhost = args.needs.offhost.find((row) => row.keyId === id); + const dump = args.needs.dumps.find((row) => row.keyId === id); return { kind: "application" as const, id, @@ -241,6 +255,13 @@ export function applicationKitKeys(args: { args.needs.stored.find((row) => row.keyId === id)?.copies ?? 0, offhostUntil: offhost && needed.has(id) ? (offhost.neededUntil ?? "unknown") : null, + localDumps: dump?.local ?? 0, + offhostDumpsUntil: + dump?.offhost && + (dump.offhost.neededUntil === null || + Date.parse(dump.offhost.neededUntil) > +args.now) + ? (dump.offhost.neededUntil ?? "unknown") + : null, }, problem: fingerprint === null ? ("not_configured" as const) : null, }; @@ -248,7 +269,14 @@ export function applicationKitKeys(args: { } function noNeed(active: boolean): KitKeyNeed { - return { active, configured: true, storedCopies: 0, offhostUntil: null }; + return { + active, + configured: true, + storedCopies: 0, + offhostUntil: null, + localDumps: 0, + offhostDumpsUntil: null, + }; } /** Pure: the off-host keys, from the two variables as the operator set them. */ diff --git a/tests/integration/backup-key-needs.test.ts b/tests/integration/backup-key-needs.test.ts index 206c38e97..3aeb4179c 100644 --- a/tests/integration/backup-key-needs.test.ts +++ b/tests/integration/backup-key-needs.test.ts @@ -124,3 +124,96 @@ describe("the keys stored backups still need", () => { expect(body.safeToDropRetiredKeys).toBe(false); }); }); + +describe("the keys the database dumps still need", () => { + // Every row rotated and no stored copy on the old key: the view called the + // old key safe to drop while the dumps from before the rotation, the + // pre-migration one that is the way back from an update among them, could + // only be restored with it. + it("names the key a dump on this server or in the bucket was written under", async () => { + const prisma = getPrismaClient(); + const admin = await prisma.user.create({ + data: { username: "dump-admin", email: "d@example.test", role: "ADMIN" }, + }); + const session = await prisma.session.create({ + data: { userId: admin.id, expiresAt: new Date(Date.now() + 60_000) }, + }); + cookieJar.set("healthlog_session", session.id); + + const { mkdtempSync, writeFileSync, rmSync } = await import("node:fs"); + const { tmpdir } = await import("node:os"); + const { join } = await import("node:path"); + const { buildManifest, dumpBaseName } = + await import("@/lib/database-backup/dump-core.mjs"); + const dir = mkdtempSync(join(tmpdir(), "hl-key-needs-")); + const savedDir = process.env.DB_BACKUP_DIR; + process.env.DB_BACKUP_DIR = dir; + try { + // A pre-migration dump taken while "old" was the active key. + useKeys("old"); + const base = dumpBaseName( + "pre-migration", + new Date(Date.now() - 86_400_000), + ); + writeFileSync(join(dir, `${base}.dump`), "PGDMP"); + writeFileSync( + join(dir, `${base}.manifest.json`), + JSON.stringify( + buildManifest({ + kind: "pre-migration", + file: `${base}.dump`, + createdAt: new Date(), + latestMigration: null, + server: { version: "18", versionNum: 180000 }, + pgDump: null, + archive: { bytes: 5, sha256: "x", entries: 1 }, + rowCounts: {}, + env: process.env, + }), + ), + ); + useKeys("cur"); + + const { GET } = await import("@/app/api/admin/encryption/status/route"); + const read = async () => + (await (await GET()).json()).data as { + safeToDropRetiredKeys: boolean; + backups: { + dumps: Array<{ keyId: string; local: number }>; + retiredKeysStillNeeded: string[]; + }; + }; + const local = await read(); + expect(local.backups.dumps).toEqual([ + expect.objectContaining({ keyId: "old", local: 1 }), + ]); + expect(local.backups.retiredKeysStillNeeded).toEqual(["old"]); + expect(local.safeToDropRetiredKeys).toBe(false); + + // Gone from this server; its off-host copy, written under "old", is + // still in a bucket whose rule for database/ nobody could read. + rmSync(join(dir, `${base}.dump`)); + rmSync(join(dir, `${base}.manifest.json`)); + await prisma.databaseBackupRun.create({ + data: { + startedAt: new Date(Date.now() - 86_400_000), + finishedAt: new Date(Date.now() - 86_000_000), + status: "succeeded", + fileName: "healthlog-x-daily.dump", + localResult: "not_persistent", + offhostResult: "uploaded", + activeKeyId: "old", + }, + }); + const offhost = await read(); + expect(offhost.backups.dumps).toEqual([ + expect.objectContaining({ keyId: "old", local: 0 }), + ]); + expect(offhost.backups.retiredKeysStillNeeded).toEqual(["old"]); + } finally { + process.env.DB_BACKUP_DIR = savedDir ?? ""; + if (savedDir === undefined) delete process.env.DB_BACKUP_DIR; + rmSync(dir, { recursive: true, force: true }); + } + }); +}); diff --git a/tests/integration/database-backup.test.ts b/tests/integration/database-backup.test.ts index 8c889cc4c..abf2bd784 100644 --- a/tests/integration/database-backup.test.ts +++ b/tests/integration/database-backup.test.ts @@ -196,6 +196,9 @@ describe("the nightly database dump", () => { error: null, }); expect(Number(runs[0].dumpBytes)).toBe(manifest.dump.bytes); + // The key a restore of this dump needs, for the recovery kit. + expect(runs[0].activeKeyId).toBe(manifest.keys.encryption.activeKeyId); + expect(runs[0].activeKeyId).not.toBeNull(); const health = await getDatabaseBackupHealth( new Date("2026-10-20T02:00:00Z"), backupEnv(),