diff --git a/.env.example b/.env.example index 8e582cc55..60572e835 100644 --- a/.env.example +++ b/.env.example @@ -249,6 +249,7 @@ API_TOKEN_HMAC_KEY="REPLACE-WITH-OUTPUT-OF-openssl-rand-hex-32" # BACKUP_S3_REGION="auto" # "auto" for Cloudflare R2; real region for AWS S3 # BACKUP_RETENTION_DAYS="30" # the number for your bucket lifecycle rule (see above) # BACKUP_ENCRYPTION_KEY="" # 64-hex-char or base64 32-byte; SEPARATE from ENCRYPTION_KEY +# BACKUP_ENCRYPTION_PREVIOUS_KEYS="" # retired off-host keys, comma-separated, read-only during a rotation # ----------------------------------------------------------------------------- # Stored backups in the database (optional) diff --git a/.env.production.example b/.env.production.example index efd693d5a..241dc1c1b 100644 --- a/.env.production.example +++ b/.env.production.example @@ -359,13 +359,17 @@ API_TOKEN_HMAC_KEY="" # BACKUP_S3_REGION="auto" # # How long backup objects are meant to live, in days. Default 30, minimum 1. -# The worker never deletes a BACKUP object -- its grant is PutObject, GetObject -# and AbortMultipartUpload -- so this number is what you set the bucket's own -# lifecycle rule to. The one key it does delete is its own connection test's -# `_healthcheck/.bin`, and that delete is best-effort: without DeleteObject -# on that prefix every Test-connection press leaves one 1-byte orphan behind, -# silently. See docs/ops/backup-restore.md. +# The app does not expire objects by age: this number is what you set the +# bucket's own lifecycle rule to. It does delete objects in two cases, so the +# credential needs DeleteObject: every copy of an account that is deleted or +# whose data is wiped, and its own connection test's `_healthcheck/.bin`. +# See docs/ops/backup-restore.md. # BACKUP_RETENTION_DAYS="30" +# +# Off-host keys no longer written with, comma-separated, kept so older +# objects stay readable during a rotation of BACKUP_ENCRYPTION_KEY. Remove a +# key once the lifecycle rule has retired every object written under it. +# BACKUP_ENCRYPTION_PREVIOUS_KEYS="" # ----------------------------------------------------------------------------- # Stored backups in the database (optional) diff --git a/docker-compose.yml b/docker-compose.yml index 6abfe4a07..4d8528dec 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -133,6 +133,7 @@ x-healthlog-env: &healthlog_env # web = serve HTTP only; pair with a separate app-worker service HEALTHLOG_PROCESS_TYPE: "${HEALTHLOG_PROCESS_TYPE:-all}" BACKUP_ENCRYPTION_KEY: "${BACKUP_ENCRYPTION_KEY:-}" + BACKUP_ENCRYPTION_PREVIOUS_KEYS: "${BACKUP_ENCRYPTION_PREVIOUS_KEYS:-}" BACKUP_S3_ENDPOINT: "${BACKUP_S3_ENDPOINT:-}" BACKUP_S3_BUCKET: "${BACKUP_S3_BUCKET:-}" BACKUP_S3_ACCESS_KEY: "${BACKUP_S3_ACCESS_KEY:-}" diff --git a/docs/api/openapi.yaml b/docs/api/openapi.yaml index 3187532b4..e5be0cb5f 100644 --- a/docs/api/openapi.yaml +++ b/docs/api/openapi.yaml @@ -13353,6 +13353,14 @@ paths: schema: $ref: "#/components/schemas/ErrorEnvelope" "401": *a1 + "409": + description: An import for this account is already queued or running, and these bytes are not the ones it carries (the + same bytes resolve to that job as a 202). `meta.errorCode` is `import.apple_health.busy` and `meta.jobId` + names the running job; poll it, then upload again. The upload was discarded. + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorEnvelope" "413": description: The declared `Content-Length` exceeds 1.5 GB. A body that exceeds the cap without declaring it is refused by the streaming sink instead, which surfaces as the 422 below. @@ -13374,8 +13382,8 @@ paths: schema: $ref: "#/components/schemas/ErrorEnvelope" "503": - description: No background worker is bound to this process, so the job cannot be queued. The upload was staged and is - discarded; retry once a worker is running. + description: No background worker is bound to this process, or the queue did not accept the job, so nothing will run it. + The staged upload is discarded (and a created job is marked failed); retry once a worker is running. content: application/json: schema: @@ -31389,31 +31397,31 @@ components: anamnesis.fact.invalidValue; auth — auth.admin.required, auth.already_authenticated, auth.mfa.code_invalid, auth.missing, auth.refresh.invalid, auth.refresh.reuse, auth.refresh.revoked, auth.reproof.failed, auth.reproof.required, auth.reproof.too_weak, auth.scope.insufficient, - auth.stepup.required, auth.token.expired, auth.token.invalid; backup — backup.payload.undecryptable, - backup.restore.active, backup.section.missing, backup.upload.too_large; body-site — body-site.invalid; - coach — coach.budget.exceeded, coach.conversation.invalidTitle, coach.conversation.notFound, - coach.fenced.attachmentConflict, coach.fenced.attachmentLimit, coach.fenced.attachmentNotFound, - coach.fenced.attachmentNotIndexed, coach.fenced.attachmentRequired, coach.fenced.attachmentUnavailable, - coach.fenced.invalid, coach.request.invalid; coach-prefs — coach-prefs.body.invalid_json, - coach-prefs.body.invalid_shape; consent — consent.ai.required; cycle — cycle.bulk.invalid, - cycle.bulk.too_large, cycle.calendar.invalid, cycle.calendar.range, cycle.cycles.invalid, - cycle.day-log.conflict, cycle.day-log.invalid, cycle.disabled, cycle.period.invalid, - cycle.symptom.custom.invalid, cycle.symptom.custom.limit; cycle-prefs — cycle-prefs.body.invalid_json, - cycle-prefs.invalid; disable-coach — disable-coach.body.invalid_json; documents — - documents.chat.conversationNotFound, documents.chat.invalid, documents.inbound.alreadyConfirmed, - documents.inbound.alreadyPartlyConfirmed, documents.inbound.budgetExceeded, - documents.inbound.decryptFailed, documents.inbound.encounterNotFound, documents.inbound.episodeNotFound, - documents.inbound.extractFailed, documents.inbound.factNotFound, documents.inbound.factNotPending, - documents.inbound.factTypeMismatch, documents.inbound.fileTooLarge, documents.inbound.fileType, - documents.inbound.invalidMetadata, documents.inbound.invalidQuery, documents.inbound.invalidUpdate, - documents.inbound.localOcrDisabled, documents.inbound.notFound, documents.inbound.notIndexed, - documents.inbound.pdfNeedsAnthropic, documents.inbound.providerUnsupported, - documents.inbound.quotaExceeded, documents.inbound.rateLimited, documents.inbound.restoreDuplicate, - documents.inbound.restoreGone, documents.inbound.sourceAliasLimit, documents.inbound.uploadBusy, - documents.inbound.uploadTimeout, documents.inbound.vaccinationNotFound, documents.sources.authRefused, - documents.sources.badResponse, documents.sources.browserOnly, documents.sources.invalidAddress, - documents.sources.linkTargetNotFound, documents.sources.notConnected, documents.sources.notFound, - documents.sources.organizationRequired, documents.sources.originNotAllowed, + auth.stepup.required, auth.token.expired, auth.token.invalid; backup — backup.foreign_reference, + backup.key.missing, backup.payload.undecryptable, backup.restore.active, backup.section.missing, + backup.upload.too_large; body-site — body-site.invalid; coach — coach.budget.exceeded, + coach.conversation.invalidTitle, coach.conversation.notFound, coach.fenced.attachmentConflict, + coach.fenced.attachmentLimit, coach.fenced.attachmentNotFound, coach.fenced.attachmentNotIndexed, + coach.fenced.attachmentRequired, coach.fenced.attachmentUnavailable, coach.fenced.invalid, + coach.request.invalid; coach-prefs — coach-prefs.body.invalid_json, coach-prefs.body.invalid_shape; + consent — consent.ai.required; cycle — cycle.bulk.invalid, cycle.bulk.too_large, cycle.calendar.invalid, + cycle.calendar.range, cycle.cycles.invalid, cycle.day-log.conflict, cycle.day-log.invalid, + cycle.disabled, cycle.period.invalid, cycle.symptom.custom.invalid, cycle.symptom.custom.limit; + cycle-prefs — cycle-prefs.body.invalid_json, cycle-prefs.invalid; disable-coach — + disable-coach.body.invalid_json; documents — documents.chat.conversationNotFound, + documents.chat.invalid, documents.inbound.alreadyConfirmed, documents.inbound.alreadyPartlyConfirmed, + documents.inbound.budgetExceeded, documents.inbound.decryptFailed, documents.inbound.encounterNotFound, + documents.inbound.episodeNotFound, documents.inbound.extractFailed, documents.inbound.factNotFound, + documents.inbound.factNotPending, documents.inbound.factTypeMismatch, documents.inbound.fileTooLarge, + documents.inbound.fileType, documents.inbound.invalidMetadata, documents.inbound.invalidQuery, + documents.inbound.invalidUpdate, documents.inbound.localOcrDisabled, documents.inbound.notFound, + documents.inbound.notIndexed, documents.inbound.pdfNeedsAnthropic, + documents.inbound.providerUnsupported, documents.inbound.quotaExceeded, documents.inbound.rateLimited, + documents.inbound.restoreDuplicate, documents.inbound.restoreGone, documents.inbound.sourceAliasLimit, + documents.inbound.uploadBusy, documents.inbound.uploadTimeout, documents.inbound.vaccinationNotFound, + documents.sources.authRefused, documents.sources.badResponse, documents.sources.browserOnly, + documents.sources.invalidAddress, documents.sources.linkTargetNotFound, documents.sources.notConnected, + documents.sources.notFound, documents.sources.organizationRequired, documents.sources.originNotAllowed, documents.sources.permissionMissing, documents.sources.rateLimited, documents.sources.redirected, documents.sources.tokenRequired, documents.sources.unavailable, documents.sources.unreachable, documents.sources.versionTooOld; encounter — encounter.invalid, encounter.link.invalid, @@ -31423,13 +31431,13 @@ components: feedback — feedback.body.invalid; health_score_config — health_score_config.invalid, health_score_config.too_narrow; illness — illness.day-log.invalid, illness.disabled, illness.episode.chronic-no-resolve, illness.episode.invalid, illness.episode.invalid-window, - illness.episode.parent-not-found, illness.insights.invalid; import — import.write_failed; ingest — - ingest.rate_limited; insights — insights.generate.budgetExceeded, insights.generate.outboundScreened, - insights.generate.profileScopeChanged, insights.generate.scopeChanged; labs — - labs.biomarker.create.invalid, labs.biomarker.duplicate, labs.biomarker.notFound, - labs.biomarker.update.invalid, labs.biomarker.update.referenceRangeInvalid, labs.create.invalid, - labs.ocr.budgetExceeded, labs.ocr.extractFailed, labs.ocr.fileTooLarge, labs.ocr.fileType, - labs.ocr.localOcrDisabled, labs.ocr.pdfNeedsAnthropic, labs.ocr.providerUnsupported, + illness.episode.parent-not-found, illness.insights.invalid; import — import.apple_health.busy, + import.write_failed; ingest — ingest.rate_limited; insights — insights.generate.budgetExceeded, + insights.generate.outboundScreened, insights.generate.profileScopeChanged, + insights.generate.scopeChanged; labs — labs.biomarker.create.invalid, labs.biomarker.duplicate, + labs.biomarker.notFound, labs.biomarker.update.invalid, labs.biomarker.update.referenceRangeInvalid, + labs.create.invalid, labs.ocr.budgetExceeded, labs.ocr.extractFailed, labs.ocr.fileTooLarge, + labs.ocr.fileType, labs.ocr.localOcrDisabled, labs.ocr.pdfNeedsAnthropic, labs.ocr.providerUnsupported, labs.ocr.rateLimited, labs.restore.invalid, labs.result.notFound, labs.update.invalid, labs.update.linkedFieldsImmutable, labs.update.numericExpected, labs.update.qualitativeExpected, labs.update.referenceRangeInvalid, labs.update.sourceReferenceRangeInvalid; managed_profile — @@ -73040,21 +73048,26 @@ components: type: string description: "Stable reason: `backup_not_found`, `backup_changed`, `backup.payload.undecryptable`, `schema_invalid`, `incompatible_schema_version`, `owner_mismatch`, `owner_not_found`, `backup.section.missing`, - `document_ciphertext_missing`, `time_budget`, `transaction_failed`, `interrupted`, `not_started`, - `enqueue_failed`, `failed_after_commit`, `unexpected`. Every code but `failed_after_commit` means the - account was not changed." + `document_ciphertext_missing`, `backup.key.missing`, `backup.foreign_reference`, `time_budget`, + `transaction_failed`, `interrupted`, `not_started`, `enqueue_failed`, `failed_after_commit`, `unexpected`. + Every code but `failed_after_commit` means the account was not changed." message: type: string sections: type: array items: type: string + keyIds: + type: array + items: + type: string required: - code - message additionalProperties: false description: Why the restore did not happen, in a sentence an operator can act on. `sections` names the missing sections - for `backup.section.missing`. + for `backup.section.missing`; `keyIds` names the encryption key ids the file needs and this server cannot open + it with, for `backup.key.missing`. AdminBackupRestoreJobResponse: type: object properties: diff --git a/docs/ops/backup-restore.md b/docs/ops/backup-restore.md index 4a975bece..4d0a70f0e 100644 --- a/docs/ops/backup-restore.md +++ b/docs/ops/backup-restore.md @@ -16,13 +16,23 @@ container (queue `data-backup-offhost`). Object key layout: ``` magic = "HLBK" (4 bytes, ASCII) -version = 0x03 (1 byte) +version = 0x04 (1 byte) +keyIdLen, keyId (1 byte + 12 ASCII hex: the first 12 hex digits of SHA-256(key)) iv = 12 random bytes (AES-GCM nonce) -ciphertext = N bytes (AES-256-GCM over gzip(JSON dump), key = BACKUP_ENCRYPTION_KEY) +ciphertext = N bytes (AES-256-GCM over gzip(JSON dump), key = BACKUP_ENCRYPTION_KEY, + associated data = "healthlog/offhost-backup/v4|") authTag = 16 bytes (AES-GCM tag, trailing) ``` -Three versions exist in the wild and all three restore. `0x01` encrypted the +Version `0x04` (from v1.39.3) adds two things. The key id says which off-host +key an object needs, so a reader holding a retired key as well finds the right +one. The associated data is the object's own key in the bucket, +`YYYY-MM-DD/user-.json.enc`: an object copied to another account's name or +another date does not decrypt, where before it opened as if it belonged there. +The restore script passes the key it downloaded, so nothing changes for an +operator. + +Four versions exist in the wild and all four restore. `0x01` encrypted the JSON directly and `0x02` gzipped it first; both carry the tag in FRONT of the ciphertext, which is what made them impossible to write a piece at a time — GCM only produces the tag once the last block is in, so a leading tag means the @@ -33,32 +43,59 @@ the restore script needs no flag to tell them apart. ## Required env vars -| Var | Required | Notes | -| ----------------------- | -------- | -------------------------------------------------------------------- | -| `BACKUP_ENCRYPTION_KEY` | yes | 64 hex chars or 32-byte base64. **Different from `ENCRYPTION_KEY`.** | -| `BACKUP_S3_ENDPOINT` | yes | e.g. `https://.r2.cloudflarestorage.com` | -| `BACKUP_S3_BUCKET` | yes | | -| `BACKUP_S3_ACCESS_KEY` | yes | | -| `BACKUP_S3_SECRET_KEY` | yes | | -| `BACKUP_S3_REGION` | no | defaults to `auto` (Cloudflare R2) | -| `BACKUP_RETENTION_DAYS` | no | not read by the app; the number for your bucket's lifecycle rule | +| Var | Required | Notes | +| --------------------------------- | -------- | ----------------------------------------------------------------------------------- | +| `BACKUP_ENCRYPTION_KEY` | yes | 64 hex chars or 32-byte base64. **Different from `ENCRYPTION_KEY`.** | +| `BACKUP_S3_ENDPOINT` | yes | e.g. `https://.r2.cloudflarestorage.com` | +| `BACKUP_S3_BUCKET` | yes | | +| `BACKUP_S3_ACCESS_KEY` | yes | | +| `BACKUP_S3_SECRET_KEY` | yes | | +| `BACKUP_S3_REGION` | no | defaults to `auto` (Cloudflare R2) | +| `BACKUP_RETENTION_DAYS` | no | not read by the app; the number for your bucket's lifecycle rule | +| `BACKUP_ENCRYPTION_PREVIOUS_KEYS` | no | retired off-host keys, comma-separated, read only (see "Rotating the off-host key") | ## Bucket permissions -The worker needs `PutObject`, `GetObject` and `AbortMultipartUpload`. It never -calls `DeleteObject` on a backup key, so a compromised worker cannot wipe the -history; `AbortMultipartUpload` only reaches an upload that same worker started -and is what clears the parts of a run that failed halfway. Without it, a failed -upload leaves parts that are billed and do not show in a bucket listing. On -Cloudflare R2 the **Object Read & Write** token already covers all three. +The worker needs `PutObject`, `GetObject`, `ListBucket`, `DeleteObject` and +`AbortMultipartUpload`. On Cloudflare R2 the **Object Read & Write** token +covers all of them. -## Bucket lifecycle (recommended) +`DeleteObject` is needed because the app deletes objects in two cases, and +only those: -The worker never deletes a backup object: its grant covers PutObject, -GetObject and AbortMultipartUpload only, so a compromised worker cannot -wipe the history, and nothing in the app reads `BACKUP_RETENTION_DAYS`. -Expiry is the storage provider's lifecycle rule, set to match the number -you wrote there: +- **An account is deleted, or its data is wiped.** Deleting an account, + deleting a managed profile and "Delete all data" each write a deletion + request in the same transaction as the deletion. The `offhost-backup-purge` job lists the bucket, deletes every + `YYYY-MM-DD/user-.json.enc` of that account and removes the request once + nothing of the account is left. It runs right after the deletion and again + every night at 03:40. A copy the nightly upload was writing while the account + was deleted is removed by the upload itself. +- **The connection test** deletes its own `_healthcheck/.bin`. + +The admin wipe of every account deletes nothing in the bucket. It is you +clearing your own host, the bucket is yours, and its copies are the way back +from a wipe pressed by mistake. If you are handing the host on, empty the +bucket yourself. + +Earlier versions of this page said the worker never deletes an object. That +was the design until v1.39.3, and it meant a deleted account's copies stayed +in the bucket for as long as the lifecycle rule kept them, or forever without +one. If the bucket refuses the delete (a credential without `DeleteObject`), +the request stays, the job retries it every night, and **Admin → Backups → +Off-host** shows how many deletions are outstanding and the bucket's answer. + +`AbortMultipartUpload` only reaches an upload the same worker started and is +what clears the parts of a run that failed halfway. Without it, a failed +upload leaves parts that are billed and do not show in a bucket listing. + +## Bucket lifecycle (required for retention) + +The app does not expire objects by age, and nothing in the app reads +`BACKUP_RETENTION_DAYS`. Expiry is the storage provider's lifecycle rule, set +to match the number you wrote there. Without one, every nightly copy of every +account is kept forever. **Admin → Backups → Off-host** reads the bucket's +lifecycle configuration and says so when there is none, or when the credential +may not read it (grant `GetLifecycleConfiguration` to let it check): ``` Filter: "" (all objects) @@ -343,6 +380,28 @@ in seconds having backed up nothing is that defect and not your record. ### Key rotation +Rotation reaches a stored backup's envelope and nothing inside it. The +content of a disaster-recovery backup is the database's ciphertext, carried +as it is stored, and a restore writes it back verbatim: a note, a document or +a coach message inside a copy taken before a rotation still needs the key the +rotation retired. The same holds for every copy in the off-host bucket and for +every file an operator has downloaded. So the rule is: **keep a retired +`ENCRYPTION_KEYS` entry until no backup you may still restore needs it.** +**Admin → Encryption → Keys the backups still need** lists, per key id, the +stored copies that need it with the oldest date, and when the last off-host +copy needing it expires under the bucket's lifecycle rule. Copies written +before v1.39.3 did not record their keys and are listed as such. + +Every restore, the restore preview, the admin upload and the monthly drill +check this before anything is changed: a file whose content needs a key this +server does not have (or an id whose key material differs) is refused with the +key id named, and the account is left as it was. + +Re-encrypting the content of stored backups during rotation was considered and +not done: it would mean decrypting, parsing, re-encrypting and re-writing every +copy in full, and it still could not reach the off-host copies or downloaded +files, so the rule above would be needed regardless. + `scripts/rotate-encryption-key.ts` and the rotation in the admin console re-seal every piece under the active key, in small batches, without reading the backup inside. The link between a piece, its copy and its position is @@ -367,6 +426,17 @@ read back. Restoring Monday's copy on Wednesday therefore costs the account everything it recorded on Tuesday. If the current state is worth keeping, take a fresh snapshot from the backups console before restoring the old one. +## Rotating the off-host key + +`BACKUP_ENCRYPTION_KEY` is the key new objects are written under. To rotate +it, set the new key there and move the old one into +`BACKUP_ENCRYPTION_PREVIOUS_KEYS` (comma-separated if there are several), then +restart. New objects are written under the new key; older objects stay +readable because every version-4 object names its key and older objects are +tried against each key in turn. Remove the old key once the lifecycle rule has +retired every object written under it, that is `BACKUP_RETENTION_DAYS` after +the restart. + ## Monthly restore drill (automatic) Since v1.16.4 a pg-boss job (`data-restore-drill`, cron `11 4 1 * *` — @@ -386,8 +456,13 @@ Outcomes: - **Failure** — empty bucket, fetch error, decryption failure (wrong or rotated key), malformed JSON: pages the same way. A decryption failure right after a `BACKUP_ENCRYPTION_KEY` change means the new - key cannot read the existing objects — re-encrypt or accept that - pre-rotation backups are only readable with the retired key. + key cannot read the existing objects: put the old key into + `BACKUP_ENCRYPTION_PREVIOUS_KEYS` (see "Rotating the off-host key"). +- **Content under a missing application key** — the object opens, but its + content was written under an `ENCRYPTION_KEYS` entry this server no longer + has, or under an id whose key material changed. The drill decrypts one value + per key id it finds, and pages naming the key. A restore of that object + would write back rows nobody can open; put the key back before restoring. - **Not configured** — deployments without the `BACKUP_S3_*` vars skip silently (wide-event warning only). diff --git a/docs/ops/encryption-key-rotation.md b/docs/ops/encryption-key-rotation.md index 078fff9a9..0396e2897 100644 --- a/docs/ops/encryption-key-rotation.md +++ b/docs/ops/encryption-key-rotation.md @@ -118,14 +118,34 @@ under `v1` (the synthetic id assigned to the existing `ENCRYPTION_KEY`). Restart. The legacy single-key fallback is now disconnected; only `v2` exists. -> **Backups are covered, and were not always.** `DataBackup.data` holds every -> weekly disaster-recovery snapshot and every uploaded pack, encrypted like -> everything else but under a column called `data`. It was outside the -> registry until v1.38.6, so a rotation before that release reported zero -> rows remaining without ever reading a backup. If you rotated on an older -> release and dropped the previous key, the stored backups are encrypted -> under the key you removed: put that key back into `ENCRYPTION_KEYS` and -> re-run the rotation on this release before removing it again. +> **The rows in the database are covered. The content of a backup is not.** +> Rotation re-encrypts every registered column, including `DataBackup.data` +> and the pieces in `DataBackupChunk`, so every stored copy opens under the +> new key. What it cannot change is what a disaster-recovery backup carries +> inside: the database's ciphertext as it was stored when the copy was taken, +> which a restore writes back verbatim. A copy taken before the rotation still +> needs the old key for its notes, documents and coach history, and so does +> every copy in the off-host bucket and every backup file you downloaded. +> +> So before step 4, also check **Admin → Encryption → Keys the backups still +> need**. It lists, per key id, the stored copies that need it with the oldest +> date, and when the last off-host copy needing it expires under the bucket's +> lifecycle rule. Keep the old key in `ENCRYPTION_KEYS` until it is no longer +> listed there (the weekly copy replaces itself within a week; delete old +> uploaded copies you no longer want, and wait out the off-host retention), or +> accept that those copies cannot be restored. Copies written before v1.39.3 +> did not record their keys and are listed as such: treat them as needing +> every key that existed when they were written. +> +> Dropping the key early no longer fails silently: a restore, a restore +> preview or an upload of a copy that needs a missing key is refused with the +> key id named, and nothing is changed. Putting the key back makes the copy +> restorable again. +> +> If you rotated on a release before v1.38.6 and dropped the previous key, +> the stored backups themselves are encrypted under the key you removed: put +> that key back into `ENCRYPTION_KEYS` and re-run the rotation on this release +> before removing it again. ## Adding a third key (v2 → v3) diff --git a/messages/de.json b/messages/de.json index d8e16465e..ffdd1c049 100644 --- a/messages/de.json +++ b/messages/de.json @@ -6590,7 +6590,7 @@ "delete": { "title": "Löschen oder zurücksetzen", "description": "Lösche dein Konto vollständig oder lösche deine Gesundheitsdaten und behalte das Konto.", - "backupLag": "Die Löschung entfernt deine Daten sofort. Verschlüsselte Off-Host-Backups enthalten sie weiterhin, bis sie aus dem Aufbewahrungsfenster des Betreibers herausfallen.", + "backupLag": "Die Löschung entfernt deine Daten sofort. Wenn dieser Server verschlüsselte externe Sicherungen anlegt, werden die Kopien deiner Daten gleich danach aus diesem Speicher gelöscht, oder in der nächsten Nacht, falls der Speicher nicht erreichbar ist.", "openAdvanced": "Löschen oder zurücksetzen" }, "posture": { @@ -7323,7 +7323,12 @@ "neverDetail": "Für dieses Konto ist nichts angekommen. Zuletzt geprüft {when}.", "unknownDetail": "Noch kein nächtlicher Lauf hat dieses Konto erfasst.", "age": "vor {hours} h", - "scheduleNote": "Der Upload läuft alle {hours} h. Frisch heißt innerhalb eines Laufs, fällig heißt ein Lauf blieb ohne Ergebnis, veraltet heißt zwei." + "scheduleNote": "Der Upload läuft alle {hours} h. Frisch heißt innerhalb eines Laufs, fällig heißt ein Lauf blieb ohne Ergebnis, veraltet heißt zwei.", + "lifecycleConfigured": "Der Bucket löscht Kopien nach {days} Tagen über seine eigene Lifecycle-Regel.", + "lifecycleMissing": "Der Bucket hat keine Lifecycle-Regel, jede nächtliche Kopie bleibt also für immer liegen. Lege eine Regel an, die Objekte nach BACKUP_RETENTION_DAYS löscht.", + "lifecycleUnknown": "Ob der Bucket alte Kopien löscht, ließ sich nicht lesen: Die Zugangsdaten dürfen seine Lifecycle-Regel nicht lesen. Prüfe, dass eine Regel Objekte nach BACKUP_RETENTION_DAYS löscht.", + "pendingDeletions": "Kopien von {count} gelöschten oder geleerten Konten liegen noch im Bucket und werden entfernt.", + "pendingDeletionsFailing": "Kopien von {count} gelöschten oder geleerten Konten liegen noch im Bucket, weil er das Löschen verweigert hat ({error}). Gib den Zugangsdaten DeleteObject; das Löschen wird jede Nacht wiederholt." }, "restoreStarted": "Wiederherstellung gestartet. Der Fortschritt steht über der Liste.", "restoreActive": "Für dieses Konto läuft schon eine Wiederherstellung. Warte, bis sie fertig ist.", @@ -7356,7 +7361,10 @@ "restoreFailureNotStarted": "Der Hintergrunddienst hat die Wiederherstellung nicht übernommen. Prüfe, ob er läuft.", "restoreFailureEnqueue": "Die Wiederherstellung konnte nicht an den Hintergrunddienst übergeben werden. Prüfe, ob er läuft.", "restoreFailureAfterCommit": "Die Daten wurden wiederhergestellt, aber ein Schritt danach ist fehlgeschlagen. Diagramme holen das womöglich erst beim nächsten nächtlichen Lauf nach.", - "restoreFailureUnexpected": "Die Wiederherstellung ist an einem unerwarteten Fehler gescheitert und wurde zurückgerollt. Das Server-Log nennt den Fehler." + "restoreFailureUnexpected": "Die Wiederherstellung ist an einem unerwarteten Fehler gescheitert und wurde zurückgerollt. Das Server-Log nennt den Fehler.", + "restoreFailureKeyMissing": "Die Sicherung braucht den Verschlüsselungsschlüssel {keys}, mit dem dieser Server sie nicht öffnen kann. Nimm den Schlüssel wieder in ENCRYPTION_KEYS auf, starte neu und starte die Wiederherstellung erneut.", + "restoreFailureForeignReference": "Die Datei verweist auf Einträge, die zu einem anderen Konto gehören. Es wurde nichts geschrieben.", + "previewKeyMissing": "Diese Sicherung lässt sich auf diesem Server nicht wiederherstellen: Sie braucht den Verschlüsselungsschlüssel {keys}. Nimm den Schlüssel zuerst wieder in ENCRYPTION_KEYS auf und starte neu." }, "danger-zone": { "title": "Gefahrenzone", @@ -7408,7 +7416,17 @@ "completed": "Abgeschlossen", "failed": "Fehlgeschlagen" }, - "coverageDetail": "Es werden nur Schlüssel-IDs und Anzahlen gezeigt — niemals Schlüsselmaterial." + "coverageDetail": "Es werden nur Schlüssel-IDs und Anzahlen gezeigt — niemals Schlüsselmaterial.", + "backupsTitle": "Schlüssel, die die Sicherungen noch brauchen", + "backupsDescription": "Ein alter Schlüssel bleibt nötig, solange eine Sicherung unten ihn noch aufführt.", + "backupsStoredLine": "Schlüssel {key}: {count} gespeicherte Kopien, die älteste vom {when}", + "backupsUnrecordedLine": "{count} gespeicherte Kopien stammen aus der Zeit, bevor Sicherungen ihre Schlüssel festhielten (die älteste vom {when}). Sie können jeden damals vorhandenen Schlüssel brauchen: Behalte alle alten Schlüssel, bis sie ersetzt oder gelöscht sind.", + "backupsOffhostLine": "Schlüssel {key}: Die letzte externe Kopie, die ihn braucht, kam am {when} in den Bucket; dort gebraucht bis {until}", + "backupsOffhostLineUnknown": "Schlüssel {key}: Die letzte externe Kopie, die ihn braucht, kam am {when} in den Bucket. Wie lange der Bucket Kopien behält, ist unbekannt; behalte den Schlüssel, bis diese Kopie sicher weg ist.", + "backupsNone": "Noch keine Sicherung hat die Schlüssel festgehalten, die sie braucht.", + "backupsStillNeed": "Alle Zeilen sind rotiert, aber Sicherungen brauchen noch {keys}", + "backupsUnrecordedBadge": "Alle Zeilen sind rotiert, aber ältere Sicherungen können noch einen ausgemusterten Schlüssel brauchen", + "backupsDetail": "Die Rotation verschlüsselt jede Zeile neu, aber eine Sicherung behält den Schlüssel, unter dem ihr Inhalt geschrieben wurde, und ihre Wiederherstellung braucht ihn. Behalte einen ausgemusterten Schlüssel in ENCRYPTION_KEYS, bis er unten nicht mehr aufgeführt ist." } }, "carrier": "Netzbetreiber", diff --git a/messages/en.json b/messages/en.json index 7e52ec2bb..ecf79c61e 100644 --- a/messages/en.json +++ b/messages/en.json @@ -6590,7 +6590,7 @@ "delete": { "title": "Delete or reset", "description": "Remove your account entirely, or wipe your health records while keeping the account.", - "backupLag": "Deletion removes your data immediately. Encrypted off-host backups still contain it until they roll off the operator's backup retention window.", + "backupLag": "Deletion removes your data immediately. If this server keeps encrypted off-host backups, the copies of your data are deleted from that storage right after, or on the next nightly run if the storage is unreachable.", "openAdvanced": "Delete or reset" }, "posture": { @@ -7323,7 +7323,12 @@ "neverDetail": "Nothing has landed for this account. Last checked {when}.", "unknownDetail": "No nightly run has recorded this account yet.", "age": "{hours} h ago", - "scheduleNote": "The upload runs every {hours} h. Fresh means inside one run, due means one run produced nothing, stale means two did not." + "scheduleNote": "The upload runs every {hours} h. Fresh means inside one run, due means one run produced nothing, stale means two did not.", + "lifecycleConfigured": "The bucket deletes copies after {days} days by its own lifecycle rule.", + "lifecycleMissing": "The bucket has no lifecycle rule, so every nightly copy is kept forever. Add a rule that expires objects after BACKUP_RETENTION_DAYS.", + "lifecycleUnknown": "Whether the bucket deletes old copies could not be read: the credential may not read its lifecycle rule. Check that a rule expires objects after BACKUP_RETENTION_DAYS.", + "pendingDeletions": "Copies of {count} deleted or wiped accounts are still in the bucket and are being removed.", + "pendingDeletionsFailing": "Copies of {count} deleted or wiped accounts are still in the bucket because it refused to delete them ({error}). Give the credential DeleteObject; the deletion is retried every night." }, "restoreStarted": "Restore started. Its progress shows above the list.", "restoreActive": "A restore of this account is already running. Wait for it to finish.", @@ -7356,7 +7361,10 @@ "restoreFailureNotStarted": "The background worker did not pick the restore up. Check that the worker is running.", "restoreFailureEnqueue": "The restore could not be handed to the background worker. Check that the worker is running.", "restoreFailureAfterCommit": "The data was restored, but a step after it failed. Charts may take until the next nightly run to catch up.", - "restoreFailureUnexpected": "The restore stopped on an unexpected error and was rolled back. The server log names the error." + "restoreFailureUnexpected": "The restore stopped on an unexpected error and was rolled back. The server log names the error.", + "restoreFailureKeyMissing": "The backup needs encryption key {keys}, which this server cannot open it with. Add the key back to ENCRYPTION_KEYS, restart, and start the restore again.", + "restoreFailureForeignReference": "The file points at records that belong to another account. Nothing was written.", + "previewKeyMissing": "This backup cannot be restored on this server: it needs encryption key {keys}. Add the key back to ENCRYPTION_KEYS and restart first." }, "danger-zone": { "title": "Danger Zone", @@ -7408,7 +7416,17 @@ "completed": "Completed", "failed": "Failed" }, - "coverageDetail": "Only key ids and counts are shown — never key material." + "coverageDetail": "Only key ids and counts are shown — never key material.", + "backupsTitle": "Keys the backups still need", + "backupsDescription": "A retired key stays needed while any backup below still lists it.", + "backupsStoredLine": "Key {key}: {count} stored copies, the oldest from {when}", + "backupsUnrecordedLine": "{count} stored copies were written before backups recorded their keys (the oldest on {when}). They may need any key that existed then: keep every old key until they are replaced or deleted.", + "backupsOffhostLine": "Key {key}: the last off-host copy that needs it went into the bucket on {when}; needed there until {until}", + "backupsOffhostLineUnknown": "Key {key}: the last off-host copy that needs it went into the bucket on {when}. The bucket's expiry is unknown, so keep the key until you know that copy is gone.", + "backupsNone": "No backup has recorded the keys it needs yet.", + "backupsStillNeed": "Every row is rotated, but backups still need {keys}", + "backupsUnrecordedBadge": "Every row is rotated, but older backups may still need a retired key", + "backupsDetail": "Rotation re-encrypts every row, but a backup keeps the key its content was written under, and restoring it needs that key. Keep a retired key in ENCRYPTION_KEYS until nothing below lists it." } }, "carrier": "Carrier", diff --git a/messages/es.json b/messages/es.json index 912be0c34..b17fc7abf 100644 --- a/messages/es.json +++ b/messages/es.json @@ -6590,7 +6590,7 @@ "delete": { "title": "Eliminar o restablecer", "description": "Elimina tu cuenta por completo o borra tus registros de salud conservando la cuenta.", - "backupLag": "La eliminación borra tus datos de inmediato. Las copias de seguridad cifradas externas aún los contienen hasta que caduquen según la ventana de retención del operador.", + "backupLag": "La eliminación borra tus datos de inmediato. Si este servidor guarda copias de seguridad cifradas externas, las copias de tus datos se borran de ese almacenamiento justo después, o en la siguiente ejecución nocturna si no está disponible.", "openAdvanced": "Eliminar o restablecer" }, "posture": { @@ -7323,7 +7323,12 @@ "neverDetail": "No ha llegado nada para esta cuenta. Última comprobación {when}.", "unknownDetail": "Ninguna ejecución nocturna ha registrado esta cuenta todavía.", "age": "hace {hours} h", - "scheduleNote": "La subida se ejecuta cada {hours} h. Reciente significa dentro de una ejecución, pendiente que una no produjo nada, obsoleta que fueron dos." + "scheduleNote": "La subida se ejecuta cada {hours} h. Reciente significa dentro de una ejecución, pendiente que una no produjo nada, obsoleta que fueron dos.", + "lifecycleConfigured": "El bucket borra las copias a los {days} días con su propia regla de ciclo de vida.", + "lifecycleMissing": "El bucket no tiene regla de ciclo de vida, así que cada copia nocturna se guarda para siempre. Añade una regla que caduque los objetos tras BACKUP_RETENTION_DAYS.", + "lifecycleUnknown": "No se pudo leer si el bucket borra las copias antiguas: la credencial no puede leer su regla de ciclo de vida. Comprueba que una regla caduque los objetos tras BACKUP_RETENTION_DAYS.", + "pendingDeletions": "Aún quedan en el bucket copias de {count} cuentas eliminadas o vaciadas, y se están borrando.", + "pendingDeletionsFailing": "Aún quedan en el bucket copias de {count} cuentas eliminadas o vaciadas porque rechazó borrarlas ({error}). Da a la credencial el permiso DeleteObject; el borrado se reintenta cada noche." }, "restoreStarted": "Restauración iniciada. Su progreso aparece encima de la lista.", "restoreActive": "Ya hay una restauración en curso para esta cuenta. Espera a que termine.", @@ -7356,7 +7361,10 @@ "restoreFailureNotStarted": "El proceso en segundo plano no recogió la restauración. Comprueba que está en marcha.", "restoreFailureEnqueue": "No se pudo entregar la restauración al proceso en segundo plano. Comprueba que está en marcha.", "restoreFailureAfterCommit": "Los datos se restauraron, pero falló un paso posterior. Puede que los gráficos no se pongan al día hasta la próxima ejecución nocturna.", - "restoreFailureUnexpected": "La restauración se detuvo por un error inesperado y se deshizo. El registro del servidor indica el error." + "restoreFailureUnexpected": "La restauración se detuvo por un error inesperado y se deshizo. El registro del servidor indica el error.", + "restoreFailureKeyMissing": "La copia necesita la clave de cifrado {keys}, con la que este servidor no puede abrirla. Vuelve a añadir la clave a ENCRYPTION_KEYS, reinicia e inicia de nuevo la restauración.", + "restoreFailureForeignReference": "El archivo apunta a registros que pertenecen a otra cuenta. No se escribió nada.", + "previewKeyMissing": "Esta copia no se puede restaurar en este servidor: necesita la clave de cifrado {keys}. Vuelve a añadir la clave a ENCRYPTION_KEYS y reinicia primero." }, "danger-zone": { "title": "Zona de peligro", @@ -7408,7 +7416,17 @@ "completed": "Completado", "failed": "Fallido" }, - "coverageDetail": "Solo se muestran los id de clave y los recuentos, nunca el material de la clave." + "coverageDetail": "Solo se muestran los id de clave y los recuentos, nunca el material de la clave.", + "backupsTitle": "Claves que las copias aún necesitan", + "backupsDescription": "Una clave retirada sigue siendo necesaria mientras alguna copia de abajo la indique.", + "backupsStoredLine": "Clave {key}: {count} copias guardadas, la más antigua del {when}", + "backupsUnrecordedLine": "{count} copias guardadas se escribieron antes de que las copias registraran sus claves (la más antigua el {when}). Pueden necesitar cualquier clave que existiera entonces: conserva todas las claves antiguas hasta que se sustituyan o borren.", + "backupsOffhostLine": "Clave {key}: la última copia externa que la necesita entró en el bucket el {when}; necesaria allí hasta el {until}", + "backupsOffhostLineUnknown": "Clave {key}: la última copia externa que la necesita entró en el bucket el {when}. No se sabe cuándo caducan las copias en el bucket, así que conserva la clave hasta saber que esa copia ya no está.", + "backupsNone": "Ninguna copia ha registrado todavía las claves que necesita.", + "backupsStillNeed": "Todas las filas están rotadas, pero las copias aún necesitan {keys}", + "backupsUnrecordedBadge": "Todas las filas están rotadas, pero copias antiguas pueden necesitar aún una clave retirada", + "backupsDetail": "La rotación vuelve a cifrar cada fila, pero una copia conserva la clave con la que se escribió su contenido, y restaurarla necesita esa clave. Mantén una clave retirada en ENCRYPTION_KEYS hasta que no aparezca abajo." } }, "carrier": "Operador", diff --git a/messages/fr.json b/messages/fr.json index e7323049e..8a7301258 100644 --- a/messages/fr.json +++ b/messages/fr.json @@ -6590,7 +6590,7 @@ "delete": { "title": "Supprimer ou réinitialiser", "description": "Supprimez complètement votre compte, ou effacez vos données de santé tout en conservant le compte.", - "backupLag": "La suppression efface vos données immédiatement. Les sauvegardes chiffrées hors site les contiennent encore jusqu'à leur expiration selon la fenêtre de conservation de l'opérateur.", + "backupLag": "La suppression efface vos données immédiatement. Si ce serveur conserve des sauvegardes externes chiffrées, les copies de vos données sont supprimées de ce stockage juste après, ou lors du prochain passage nocturne s'il est injoignable.", "openAdvanced": "Supprimer ou réinitialiser" }, "posture": { @@ -7323,7 +7323,12 @@ "neverDetail": "Rien n'est arrivé pour ce compte. Dernière vérification {when}.", "unknownDetail": "Aucun cycle nocturne n'a encore enregistré ce compte.", "age": "il y a {hours} h", - "scheduleNote": "L'envoi tourne toutes les {hours} h. Récente signifie dans un cycle, attendue qu'un cycle n'a rien produit, périmée que deux n'ont rien produit." + "scheduleNote": "L'envoi tourne toutes les {hours} h. Récente signifie dans un cycle, attendue qu'un cycle n'a rien produit, périmée que deux n'ont rien produit.", + "lifecycleConfigured": "Le bucket supprime les copies après {days} jours grâce à sa propre règle de cycle de vie.", + "lifecycleMissing": "Le bucket n'a aucune règle de cycle de vie : chaque copie nocturne est donc conservée indéfiniment. Ajoutez une règle qui fait expirer les objets après BACKUP_RETENTION_DAYS.", + "lifecycleUnknown": "Impossible de savoir si le bucket supprime les anciennes copies : l'identifiant ne peut pas lire sa règle de cycle de vie. Vérifiez qu'une règle fait expirer les objets après BACKUP_RETENTION_DAYS.", + "pendingDeletions": "Des copies de {count} comptes supprimés ou vidés sont encore dans le bucket et sont en cours de suppression.", + "pendingDeletionsFailing": "Des copies de {count} comptes supprimés ou vidés sont encore dans le bucket, qui a refusé de les supprimer ({error}). Accordez DeleteObject à l'identifiant ; la suppression est retentée chaque nuit." }, "restoreStarted": "Restauration lancée. Sa progression s'affiche au-dessus de la liste.", "restoreActive": "Une restauration de ce compte est déjà en cours. Attends qu'elle se termine.", @@ -7356,7 +7361,10 @@ "restoreFailureNotStarted": "Le processus en arrière-plan n'a pas pris la restauration en charge. Vérifie qu'il tourne.", "restoreFailureEnqueue": "La restauration n'a pas pu être transmise au processus en arrière-plan. Vérifie qu'il tourne.", "restoreFailureAfterCommit": "Les données ont été restaurées, mais une étape suivante a échoué. Les graphiques ne seront peut-être à jour qu'après la prochaine exécution nocturne.", - "restoreFailureUnexpected": "La restauration s'est arrêtée sur une erreur inattendue et a été annulée. Le journal du serveur indique l'erreur." + "restoreFailureUnexpected": "La restauration s'est arrêtée sur une erreur inattendue et a été annulée. Le journal du serveur indique l'erreur.", + "restoreFailureKeyMissing": "La sauvegarde a besoin de la clé de chiffrement {keys}, avec laquelle ce serveur ne peut pas l'ouvrir. Remettez la clé dans ENCRYPTION_KEYS, redémarrez, puis relancez la restauration.", + "restoreFailureForeignReference": "Le fichier renvoie à des éléments qui appartiennent à un autre compte. Rien n'a été écrit.", + "previewKeyMissing": "Cette sauvegarde ne peut pas être restaurée sur ce serveur : elle a besoin de la clé de chiffrement {keys}. Remettez d'abord la clé dans ENCRYPTION_KEYS et redémarrez." }, "danger-zone": { "title": "Zone de danger", @@ -7408,7 +7416,17 @@ "completed": "Terminé", "failed": "Échoué" }, - "coverageDetail": "Seuls les id de clé et les décomptes sont affichés, jamais le matériel de clé." + "coverageDetail": "Seuls les id de clé et les décomptes sont affichés, jamais le matériel de clé.", + "backupsTitle": "Clés dont les sauvegardes ont encore besoin", + "backupsDescription": "Une clé retirée reste nécessaire tant qu'une sauvegarde ci-dessous la mentionne.", + "backupsStoredLine": "Clé {key} : {count} copies stockées, la plus ancienne du {when}", + "backupsUnrecordedLine": "{count} copies stockées ont été écrites avant que les sauvegardes n'enregistrent leurs clés (la plus ancienne le {when}). Elles peuvent avoir besoin de n'importe quelle clé de l'époque : gardez toutes les anciennes clés jusqu'à leur remplacement ou suppression.", + "backupsOffhostLine": "Clé {key} : la dernière copie externe qui en a besoin est arrivée dans le bucket le {when} ; nécessaire jusqu'au {until}", + "backupsOffhostLineUnknown": "Clé {key} : la dernière copie externe qui en a besoin est arrivée dans le bucket le {when}. La durée de conservation du bucket est inconnue : gardez la clé jusqu'à être sûr que cette copie a disparu.", + "backupsNone": "Aucune sauvegarde n'a encore enregistré les clés dont elle a besoin.", + "backupsStillNeed": "Toutes les lignes sont renouvelées, mais des sauvegardes ont encore besoin de {keys}", + "backupsUnrecordedBadge": "Toutes les lignes sont renouvelées, mais d'anciennes sauvegardes peuvent encore avoir besoin d'une clé retirée", + "backupsDetail": "La rotation rechiffre chaque ligne, mais une sauvegarde garde la clé sous laquelle son contenu a été écrit, et sa restauration en a besoin. Gardez une clé retirée dans ENCRYPTION_KEYS tant qu'elle figure ci-dessous." } }, "carrier": "Opérateur", diff --git a/messages/it.json b/messages/it.json index c2345d3fa..f9c09d238 100644 --- a/messages/it.json +++ b/messages/it.json @@ -6590,7 +6590,7 @@ "delete": { "title": "Elimina o reimposta", "description": "Elimina completamente il tuo account oppure cancella i tuoi dati sanitari mantenendo l'account.", - "backupLag": "L'eliminazione rimuove subito i tuoi dati. I backup cifrati esterni li contengono ancora finché non scadono secondo la finestra di conservazione dell'operatore.", + "backupLag": "L'eliminazione rimuove subito i tuoi dati. Se questo server conserva backup esterni cifrati, le copie dei tuoi dati vengono eliminate da quello spazio subito dopo, o alla successiva esecuzione notturna se non è raggiungibile.", "openAdvanced": "Elimina o reimposta" }, "posture": { @@ -7323,7 +7323,12 @@ "neverDetail": "Per questo account non è arrivato nulla. Ultimo controllo {when}.", "unknownDetail": "Nessun giro notturno ha ancora registrato questo account.", "age": "{hours} h fa", - "scheduleNote": "Il caricamento gira ogni {hours} h. Recente significa entro un giro, attesa che un giro non ha prodotto nulla, obsoleta che non lo hanno fatto due." + "scheduleNote": "Il caricamento gira ogni {hours} h. Recente significa entro un giro, attesa che un giro non ha prodotto nulla, obsoleta che non lo hanno fatto due.", + "lifecycleConfigured": "Il bucket elimina le copie dopo {days} giorni con la propria regola di ciclo di vita.", + "lifecycleMissing": "Il bucket non ha una regola di ciclo di vita, quindi ogni copia notturna viene conservata per sempre. Aggiungi una regola che faccia scadere gli oggetti dopo BACKUP_RETENTION_DAYS.", + "lifecycleUnknown": "Non è stato possibile leggere se il bucket elimina le copie vecchie: la credenziale non può leggere la sua regola di ciclo di vita. Verifica che una regola faccia scadere gli oggetti dopo BACKUP_RETENTION_DAYS.", + "pendingDeletions": "Copie di {count} account eliminati o svuotati sono ancora nel bucket e vengono rimosse.", + "pendingDeletionsFailing": "Copie di {count} account eliminati o svuotati sono ancora nel bucket perché ha rifiutato di eliminarle ({error}). Concedi DeleteObject alla credenziale; l'eliminazione viene ritentata ogni notte." }, "restoreStarted": "Ripristino avviato. L'avanzamento compare sopra l'elenco.", "restoreActive": "Per questo account è già in corso un ripristino. Aspetta che finisca.", @@ -7356,7 +7361,10 @@ "restoreFailureNotStarted": "Il processo in background non ha preso in carico il ripristino. Controlla che sia in esecuzione.", "restoreFailureEnqueue": "Non è stato possibile passare il ripristino al processo in background. Controlla che sia in esecuzione.", "restoreFailureAfterCommit": "I dati sono stati ripristinati, ma un passaggio successivo non è riuscito. I grafici potrebbero aggiornarsi solo con la prossima esecuzione notturna.", - "restoreFailureUnexpected": "Il ripristino si è fermato per un errore imprevisto ed è stato annullato. Il log del server indica l'errore." + "restoreFailureUnexpected": "Il ripristino si è fermato per un errore imprevisto ed è stato annullato. Il log del server indica l'errore.", + "restoreFailureKeyMissing": "Il backup richiede la chiave di cifratura {keys}, con cui questo server non riesce ad aprirlo. Aggiungi di nuovo la chiave a ENCRYPTION_KEYS, riavvia e avvia di nuovo il ripristino.", + "restoreFailureForeignReference": "Il file rimanda a voci che appartengono a un altro account. Non è stato scritto nulla.", + "previewKeyMissing": "Questo backup non può essere ripristinato su questo server: richiede la chiave di cifratura {keys}. Aggiungi prima di nuovo la chiave a ENCRYPTION_KEYS e riavvia." }, "danger-zone": { "title": "Zona pericolosa", @@ -7408,7 +7416,17 @@ "completed": "Completato", "failed": "Fallito" }, - "coverageDetail": "Vengono mostrati solo gli id di chiave e i conteggi, mai il materiale della chiave." + "coverageDetail": "Vengono mostrati solo gli id di chiave e i conteggi, mai il materiale della chiave.", + "backupsTitle": "Chiavi di cui i backup hanno ancora bisogno", + "backupsDescription": "Una chiave ritirata resta necessaria finché un backup qui sotto la elenca ancora.", + "backupsStoredLine": "Chiave {key}: {count} copie salvate, la più vecchia del {when}", + "backupsUnrecordedLine": "{count} copie salvate sono state scritte prima che i backup registrassero le loro chiavi (la più vecchia il {when}). Possono richiedere qualsiasi chiave esistente allora: conserva tutte le vecchie chiavi finché non vengono sostituite o eliminate.", + "backupsOffhostLine": "Chiave {key}: l'ultima copia esterna che la richiede è entrata nel bucket il {when}; necessaria fino al {until}", + "backupsOffhostLineUnknown": "Chiave {key}: l'ultima copia esterna che la richiede è entrata nel bucket il {when}. La scadenza del bucket è sconosciuta, quindi conserva la chiave finché non sai che quella copia non c'è più.", + "backupsNone": "Nessun backup ha ancora registrato le chiavi di cui ha bisogno.", + "backupsStillNeed": "Tutte le righe sono ruotate, ma i backup hanno ancora bisogno di {keys}", + "backupsUnrecordedBadge": "Tutte le righe sono ruotate, ma backup più vecchi potrebbero ancora richiedere una chiave ritirata", + "backupsDetail": "La rotazione cifra di nuovo ogni riga, ma un backup conserva la chiave con cui è stato scritto il suo contenuto, e ripristinarlo richiede quella chiave. Tieni una chiave ritirata in ENCRYPTION_KEYS finché compare qui sotto." } }, "carrier": "Operatore", diff --git a/messages/ko.json b/messages/ko.json index 74e4d5285..36c86ca0b 100644 --- a/messages/ko.json +++ b/messages/ko.json @@ -6590,7 +6590,7 @@ "delete": { "title": "삭제 또는 초기화", "description": "계정을 완전히 없애거나, 계정은 두고 건강 기록만 지워요.", - "backupLag": "삭제하면 데이터는 즉시 지워져요. 외부에 보관된 암호화 백업에는 운영자의 백업 보관 기간이 끝날 때까지 남아 있어요.", + "backupLag": "삭제하면 데이터는 즉시 지워져요. 이 서버가 암호화된 외부 백업을 보관한다면 그 저장소에 있는 데이터 사본도 곧바로 삭제되고, 저장소에 연결할 수 없으면 다음 야간 실행 때 삭제돼요.", "openAdvanced": "삭제 또는 초기화" }, "posture": { @@ -7302,7 +7302,12 @@ "neverDetail": "이 계정으로 도착한 것이 없습니다. 마지막 확인 {when}.", "unknownDetail": "아직 어떤 야간 실행도 이 계정을 기록하지 않았습니다.", "age": "{hours}시간 전", - "scheduleNote": "업로드는 {hours}시간마다 실행됩니다. 최신은 한 번의 실행 이내, 지연은 한 번이 아무것도 만들지 못한 상태, 오래됨은 두 번이 그런 상태입니다." + "scheduleNote": "업로드는 {hours}시간마다 실행됩니다. 최신은 한 번의 실행 이내, 지연은 한 번이 아무것도 만들지 못한 상태, 오래됨은 두 번이 그런 상태입니다.", + "lifecycleConfigured": "버킷의 수명 주기 규칙에 따라 복사본이 {days}일 후 삭제됩니다.", + "lifecycleMissing": "버킷에 수명 주기 규칙이 없어 매일 밤의 복사본이 영구히 보관됩니다. BACKUP_RETENTION_DAYS 후 객체를 만료시키는 규칙을 추가하세요.", + "lifecycleUnknown": "버킷이 오래된 복사본을 삭제하는지 읽을 수 없었습니다. 자격 증명이 수명 주기 규칙을 읽지 못할 수 있습니다. BACKUP_RETENTION_DAYS 후 객체를 만료시키는 규칙이 있는지 확인하세요.", + "pendingDeletions": "삭제되었거나 비워진 계정 {count}개의 복사본이 아직 버킷에 있으며 제거 중입니다.", + "pendingDeletionsFailing": "버킷이 삭제를 거부하여 삭제되었거나 비워진 계정 {count}개의 복사본이 아직 남아 있습니다 ({error}). 자격 증명에 DeleteObject 권한을 부여하세요. 삭제는 매일 밤 다시 시도됩니다." }, "restoreStarted": "복원을 시작했어요. 진행 상황은 목록 위에 표시돼요.", "restoreActive": "이 계정의 복원이 이미 진행 중이에요. 끝날 때까지 기다려 주세요.", @@ -7335,7 +7340,10 @@ "restoreFailureNotStarted": "백그라운드 작업자가 복원을 가져가지 않았어요. 작업자가 실행 중인지 확인해 주세요.", "restoreFailureEnqueue": "복원을 백그라운드 작업자에게 넘기지 못했어요. 작업자가 실행 중인지 확인해 주세요.", "restoreFailureAfterCommit": "데이터는 복원됐지만 그다음 단계가 실패했어요. 차트는 다음 야간 실행 때 따라잡을 수 있어요.", - "restoreFailureUnexpected": "예상하지 못한 오류로 복원이 멈췄고 되돌려졌어요. 서버 로그에 오류가 나와 있어요." + "restoreFailureUnexpected": "예상하지 못한 오류로 복원이 멈췄고 되돌려졌어요. 서버 로그에 오류가 나와 있어요.", + "restoreFailureKeyMissing": "이 백업에는 암호화 키 {keys}가 필요하지만 이 서버는 그 키로 백업을 열 수 없습니다. 키를 ENCRYPTION_KEYS에 다시 추가하고 재시작한 뒤 복원을 다시 시작하세요.", + "restoreFailureForeignReference": "파일이 다른 계정에 속한 기록을 가리키고 있습니다. 아무것도 기록되지 않았습니다.", + "previewKeyMissing": "이 서버에서는 이 백업을 복원할 수 없습니다. 암호화 키 {keys}가 필요합니다. 먼저 키를 ENCRYPTION_KEYS에 다시 추가하고 재시작하세요." }, "danger-zone": { "title": "위험 구역", @@ -7387,7 +7395,17 @@ "completed": "완료", "failed": "실패" }, - "coverageDetail": "키 id와 개수만 보여줘요 — 키 자체는 절대 표시하지 않아요." + "coverageDetail": "키 id와 개수만 보여줘요 — 키 자체는 절대 표시하지 않아요.", + "backupsTitle": "백업에 아직 필요한 키", + "backupsDescription": "아래 백업 중 하나라도 표시하는 동안 폐기한 키는 계속 필요합니다.", + "backupsStoredLine": "키 {key}: 저장된 복사본 {count}개, 가장 오래된 것은 {when}", + "backupsUnrecordedLine": "저장된 복사본 {count}개는 백업이 키를 기록하기 전에 작성되었습니다(가장 오래된 것은 {when}). 당시 존재하던 어떤 키든 필요할 수 있으니, 교체되거나 삭제될 때까지 이전 키를 모두 유지하세요.", + "backupsOffhostLine": "키 {key}: 이 키가 필요한 마지막 외부 복사본이 {when}에 버킷에 저장되었습니다. {until}까지 필요합니다", + "backupsOffhostLineUnknown": "키 {key}: 이 키가 필요한 마지막 외부 복사본이 {when}에 버킷에 저장되었습니다. 버킷의 만료 기간을 알 수 없으므로 그 복사본이 사라진 것을 확인할 때까지 키를 유지하세요.", + "backupsNone": "아직 필요한 키를 기록한 백업이 없습니다.", + "backupsStillNeed": "모든 행이 회전되었지만 백업에 아직 {keys}가 필요합니다", + "backupsUnrecordedBadge": "모든 행이 회전되었지만 이전 백업에는 아직 폐기한 키가 필요할 수 있습니다", + "backupsDetail": "회전은 모든 행을 다시 암호화하지만, 백업은 내용이 기록된 키를 그대로 유지하며 복원에는 그 키가 필요합니다. 아래에 더 이상 표시되지 않을 때까지 폐기한 키를 ENCRYPTION_KEYS에 유지하세요." } }, "carrier": "통신사", diff --git a/messages/pl.json b/messages/pl.json index 95220aed4..4a7a06f52 100644 --- a/messages/pl.json +++ b/messages/pl.json @@ -6590,7 +6590,7 @@ "delete": { "title": "Usuń lub zresetuj", "description": "Usuń całe konto albo wyczyść dane zdrowotne, zachowując konto.", - "backupLag": "Usunięcie natychmiast usuwa Twoje dane. Zaszyfrowane kopie zapasowe poza serwerem nadal je zawierają, dopóki nie wygasną zgodnie z oknem retencji operatora.", + "backupLag": "Usunięcie natychmiast usuwa Twoje dane. Jeśli ten serwer przechowuje zaszyfrowane kopie zapasowe poza serwerem, kopie Twoich danych są usuwane z tego magazynu zaraz potem, a jeśli jest niedostępny, przy następnym nocnym przebiegu.", "openAdvanced": "Usuń lub zresetuj" }, "posture": { @@ -7323,7 +7323,12 @@ "neverDetail": "Dla tego konta nic nie dotarło. Ostatnie sprawdzenie {when}.", "unknownDetail": "Żaden nocny przebieg nie zarejestrował jeszcze tego konta.", "age": "{hours} h temu", - "scheduleNote": "Wysyłka działa co {hours} h. Świeża znaczy w obrębie jednego przebiegu, zaległa że jeden nic nie dał, przeterminowana że dwa." + "scheduleNote": "Wysyłka działa co {hours} h. Świeża znaczy w obrębie jednego przebiegu, zaległa że jeden nic nie dał, przeterminowana że dwa.", + "lifecycleConfigured": "Bucket usuwa kopie po {days} dniach zgodnie z własną regułą cyklu życia.", + "lifecycleMissing": "Bucket nie ma reguły cyklu życia, więc każda nocna kopia zostaje na zawsze. Dodaj regułę, która usuwa obiekty po BACKUP_RETENTION_DAYS.", + "lifecycleUnknown": "Nie udało się odczytać, czy bucket usuwa stare kopie: poświadczenia nie mogą odczytać jego reguły cyklu życia. Sprawdź, czy reguła usuwa obiekty po BACKUP_RETENTION_DAYS.", + "pendingDeletions": "Kopie {count} usuniętych lub wyczyszczonych kont są jeszcze w buckecie i są usuwane.", + "pendingDeletionsFailing": "Kopie {count} usuniętych lub wyczyszczonych kont są jeszcze w buckecie, bo odmówił ich usunięcia ({error}). Nadaj poświadczeniom uprawnienie DeleteObject; usuwanie jest ponawiane co noc." }, "restoreStarted": "Przywracanie rozpoczęte. Postęp widać nad listą.", "restoreActive": "Dla tego konta trwa już przywracanie. Poczekaj, aż się skończy.", @@ -7356,7 +7361,10 @@ "restoreFailureNotStarted": "Proces w tle nie przejął przywracania. Sprawdź, czy działa.", "restoreFailureEnqueue": "Nie udało się przekazać przywracania do procesu w tle. Sprawdź, czy działa.", "restoreFailureAfterCommit": "Dane zostały przywrócone, ale krok po nich się nie powiódł. Wykresy mogą nadrobić to dopiero przy następnym nocnym przebiegu.", - "restoreFailureUnexpected": "Przywracanie zatrzymało się na nieoczekiwanym błędzie i zostało wycofane. Dziennik serwera podaje błąd." + "restoreFailureUnexpected": "Przywracanie zatrzymało się na nieoczekiwanym błędzie i zostało wycofane. Dziennik serwera podaje błąd.", + "restoreFailureKeyMissing": "Kopia wymaga klucza szyfrowania {keys}, którym ten serwer nie może jej otworzyć. Dodaj klucz z powrotem do ENCRYPTION_KEYS, uruchom ponownie i rozpocznij przywracanie jeszcze raz.", + "restoreFailureForeignReference": "Plik wskazuje na wpisy należące do innego konta. Nic nie zostało zapisane.", + "previewKeyMissing": "Tej kopii nie da się przywrócić na tym serwerze: wymaga klucza szyfrowania {keys}. Najpierw dodaj klucz z powrotem do ENCRYPTION_KEYS i uruchom ponownie." }, "danger-zone": { "title": "Strefa zagrożenia", @@ -7408,7 +7416,17 @@ "completed": "Zakończono", "failed": "Niepowodzenie" }, - "coverageDetail": "Pokazywane są tylko identyfikatory kluczy i liczby — nigdy materiał klucza." + "coverageDetail": "Pokazywane są tylko identyfikatory kluczy i liczby — nigdy materiał klucza.", + "backupsTitle": "Klucze, których kopie nadal potrzebują", + "backupsDescription": "Wycofany klucz jest potrzebny, dopóki którakolwiek kopia poniżej go wymienia.", + "backupsStoredLine": "Klucz {key}: {count} zapisanych kopii, najstarsza z {when}", + "backupsUnrecordedLine": "{count} zapisanych kopii powstało, zanim kopie zaczęły zapisywać swoje klucze (najstarsza {when}). Mogą potrzebować każdego klucza, który wtedy istniał: zachowaj wszystkie stare klucze, dopóki nie zostaną zastąpione lub usunięte.", + "backupsOffhostLine": "Klucz {key}: ostatnia kopia zewnętrzna, która go potrzebuje, trafiła do bucketu {when}; potrzebny tam do {until}", + "backupsOffhostLineUnknown": "Klucz {key}: ostatnia kopia zewnętrzna, która go potrzebuje, trafiła do bucketu {when}. Nie wiadomo, jak długo bucket przechowuje kopie, więc zachowaj klucz, dopóki nie masz pewności, że tej kopii już nie ma.", + "backupsNone": "Żadna kopia nie zapisała jeszcze kluczy, których potrzebuje.", + "backupsStillNeed": "Wszystkie wiersze są zrotowane, ale kopie nadal potrzebują {keys}", + "backupsUnrecordedBadge": "Wszystkie wiersze są zrotowane, ale starsze kopie mogą nadal potrzebować wycofanego klucza", + "backupsDetail": "Rotacja ponownie szyfruje każdy wiersz, ale kopia zachowuje klucz, którym zapisano jej zawartość, a jej przywrócenie wymaga tego klucza. Trzymaj wycofany klucz w ENCRYPTION_KEYS, dopóki widnieje poniżej." } }, "carrier": "Operator", diff --git a/prisma/migrations/0358_backup_key_ids_and_offhost_purge/migration.sql b/prisma/migrations/0358_backup_key_ids_and_offhost_purge/migration.sql new file mode 100644 index 000000000..09a577e69 --- /dev/null +++ b/prisma/migrations/0358_backup_key_ids_and_offhost_purge/migration.sql @@ -0,0 +1,37 @@ +-- Backups: which keys their content needs, and off-host copies that leave +-- with the account. +-- +-- 1. A stored backup carries encrypted columns as the stored ciphertext, and +-- key rotation cannot reach inside a copy. Each copy now records the key +-- ids its content was written under, so the admin encryption view can say +-- which retired key is still needed and by how old a copy. Copies written +-- before this release are marked unrecorded rather than assumed clean. +ALTER TABLE "data_backups" + ADD COLUMN "inner_key_ids" TEXT[] NOT NULL DEFAULT ARRAY[]::TEXT[], + ADD COLUMN "inner_key_ids_recorded" BOOLEAN NOT NULL DEFAULT false; + +-- 2. The same fact for the off-host bucket, per key rather than per object: +-- when the nightly run last wrote an object that needed each key. +CREATE TABLE "offhost_backup_key_use" ( + "key_id" TEXT NOT NULL, + "first_written_at" TIMESTAMP(3) NOT NULL, + "last_written_at" TIMESTAMP(3) NOT NULL, + CONSTRAINT "offhost_backup_key_use_pkey" PRIMARY KEY ("key_id") +); + +-- 3. Accounts whose off-host objects have to be deleted: written with the +-- account deletion or the data wipe, removed once the bucket holds nothing +-- for the account. No foreign key, because a deleted account's row is gone +-- by the time the job runs. +CREATE TABLE "offhost_purge_requests" ( + "id" TEXT NOT NULL, + "subject_id" TEXT NOT NULL, + "reason" TEXT NOT NULL, + "requested_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "attempts" INTEGER NOT NULL DEFAULT 0, + "last_attempt_at" TIMESTAMP(3), + "last_failure" TEXT, + CONSTRAINT "offhost_purge_requests_pkey" PRIMARY KEY ("id") +); +CREATE INDEX "offhost_purge_requests_requested_at_idx" + ON "offhost_purge_requests"("requested_at"); diff --git a/prisma/schema.prisma b/prisma/schema.prisma index 9ea0208a2..fbd728106 100644 --- a/prisma/schema.prisma +++ b/prisma/schema.prisma @@ -4980,19 +4980,29 @@ model PushSubscription { // ─── Data Backups ──────────────────────────────────────────── model DataBackup { - id String @id @default(cuid()) - userId String @map("user_id") - type String @default("WEEKLY_AUTO") // WEEKLY_AUTO, MANUAL_UPLOAD_ + id String @id @default(cuid()) + userId String @map("user_id") + type String @default("WEEKLY_AUTO") // WEEKLY_AUTO, MANUAL_UPLOAD_ /// The whole encrypted copy as one value: how every backup was stored up /// to v1.39.1. NULL for a copy stored in pieces (`chunks`), which is how /// every backup is written from v1.39.2 on. A row carries one or the other. - data String? + data String? /// How many pieces a chunked copy has, and the random id sealed into every /// one of them (`src/lib/export/backup-chunks.ts`). The reader checks both /// against the pieces before it releases a byte; NULL on a single-value copy. - chunkCount Int? @map("chunk_count") - chunkStreamId String? @map("chunk_stream_id") - createdAt DateTime @default(now()) @map("created_at") + chunkCount Int? @map("chunk_count") + chunkStreamId String? @map("chunk_stream_id") + /// The application key ids the ciphertext INSIDE this copy was written + /// under, recorded as the copy is written (`createBackupKeyIdTextScanner`). + /// Key rotation re-seals the copy's envelope but not what is inside it, so + /// a key stays needed while any copy lists it. NULL for a copy written + /// before v1.39.3, which the admin encryption view counts as unrecorded. + /// Consumer: `GET /api/admin/encryption/status` (`backups.stored`). + innerKeyIds String[] @default([]) @map("inner_key_ids") + /// Whether `innerKeyIds` was recorded; false on every copy written before + /// v1.39.3. Consumer: the same status read (`backups.unrecorded`). + innerKeyIdsRecorded Boolean @default(false) @map("inner_key_ids_recorded") + createdAt DateTime @default(now()) @map("created_at") user User @relation(fields: [userId], references: [id], onDelete: Cascade) chunks DataBackupChunk[] @@ -5158,6 +5168,43 @@ model OffhostBackupState { @@map("offhost_backup_state") } +/// Per application key id, when the nightly off-host run last put an object +/// in the bucket whose inner ciphertext needed that key. The bucket's +/// lifecycle rule keeps each object for its retention, so a retired key is +/// needed until that long after this date. Not per account: the question it +/// answers is about keys. Written by `runOffhostBackup`; consumer: +/// `GET /api/admin/encryption/status` (`backups.offhost`). +model OffhostBackupKeyUse { + keyId String @id @map("key_id") + firstWrittenAt DateTime @map("first_written_at") + lastWrittenAt DateTime @map("last_written_at") + + @@map("offhost_backup_key_use") +} + +/// An account whose off-host copies have to leave the bucket: it was deleted, +/// or its data was wiped. Written in the same transaction as the deletion, so +/// the request cannot be lost between the two, and deleted once every object +/// for the account is gone. `subjectId` is deliberately not a relation: for a +/// deleted account the user row is already gone. Written by +/// `requestOffhostPurge`; consumer: `processOffhostPurges` (the +/// `offhost-backup-purge` job) and the off-host card's pending count behind +/// `GET /api/admin/backups`. +model OffhostPurgeRequest { + id String @id @default(cuid()) + subjectId String @map("subject_id") + /// `account_deleted`, `data_wiped` or `managed_profile_deleted`. + reason String + requestedAt DateTime @default(now()) @map("requested_at") + attempts Int @default(0) + lastAttemptAt DateTime? @map("last_attempt_at") + /// The bucket's last refusal, trimmed; shown to the operator. + lastFailure String? @map("last_failure") + + @@index([requestedAt]) + @@map("offhost_purge_requests") +} + // ─── Apple Health export.zip import (v1.4.34) ─────────────── // // Mirrors a `pg-boss` `apple-health-import` job's lifecycle so the diff --git a/scripts/env-manifest.json b/scripts/env-manifest.json index a41698ff1..c7ac52630 100644 --- a/scripts/env-manifest.json +++ b/scripts/env-manifest.json @@ -323,6 +323,11 @@ "purpose": "Trusted reverse-proxy hop count; the client IP is read that many entries from the right of X-Forwarded-For.", "example": "1" }, + { + "name": "BACKUP_ENCRYPTION_PREVIOUS_KEYS", + "purpose": "Comma-separated off-host backup keys that are no longer written with but may still be needed to read older objects. Set during a rotation of BACKUP_ENCRYPTION_KEY until the bucket's lifecycle rule has retired every object written under the old key.", + "example": "" + }, { "name": "BACKUP_MAX_STORED_MB", "purpose": "Largest stored backup of one account the app keeps, in megabytes of encrypted copy (2048 by default). A storage limit, independent of the app's memory: a backup past it fails with a message naming this variable, and the previous copy stays.", diff --git a/scripts/restore-backup.ts b/scripts/restore-backup.ts index c6f4a51fe..15cc8e80a 100644 --- a/scripts/restore-backup.ts +++ b/scripts/restore-backup.ts @@ -32,6 +32,7 @@ import { loadOffhostConfig, getS3Client, openBackupObject, + offhostKeyRing, OffhostBackupNotConfiguredError, } from "@/lib/jobs/offhost-backup"; @@ -109,7 +110,7 @@ async function main() { `Downloading s3://${cfg.bucket}/${key} from ${cfg.endpoint} (region=${cfg.region})`, ); const ciphertext = await s3.getObject(key); - const source = openBackupObject(ciphertext, cfg.encryptionKey); + const source = openBackupObject(ciphertext, offhostKeyRing(cfg), key); // Checked first, in one streamed pass that keeps only the small sections, // so nothing is written for an object that is not a backup. diff --git a/src/app/api/admin/backups/[id]/summary/route.ts b/src/app/api/admin/backups/[id]/summary/route.ts index d8f89eb6d..b1499a1ed 100644 --- a/src/app/api/admin/backups/[id]/summary/route.ts +++ b/src/app/api/admin/backups/[id]/summary/route.ts @@ -22,9 +22,15 @@ import { storedBackupRefusal, STORED_BACKUP_SELECT, } from "@/lib/export/stored-backup"; +import { + assessBackupKeys, + BACKUP_KEY_MISSING_CODE, + describeBackupKeyProblem, +} from "@/lib/export/backup-key-ids"; import { readStreamedBackup, type BackupSource, + type StreamedBackup, } from "@/lib/export/streamed-backup"; import { annotate } from "@/lib/logging/context"; import { @@ -72,8 +78,9 @@ export const GET = apiHandler( let payload; let measurementCount: number; + let streamed: StreamedBackup; try { - const streamed = await readStreamedBackup(source); + streamed = await readStreamedBackup(source); measurementCount = streamed.measurementCount; payload = parseBackupPayload(streamed.raw); } catch (err) { @@ -93,6 +100,22 @@ export const GET = apiHandler( ); } + // The preview is where the operator decides, so it says what the restore + // would refuse: a key the file's inner ciphertext needs and this host + // does not have. The instance settings are left out here, as the restore + // leaves them out unless asked; the restore itself checks them when they + // are asked for. + const keyVerdict = assessBackupKeys(streamed.keys, { + ignoreSections: new Set(["appSettings"]), + }); + const keyProblem = describeBackupKeyProblem(keyVerdict); + if (keyProblem) { + return apiError(keyProblem, 422, { + errorCode: BACKUP_KEY_MISSING_CODE, + keyIds: [...keyVerdict.missing, ...keyVerdict.unreadable], + }); + } + const summary = { ...summarizeBackup(payload), measurements: measurementCount, diff --git a/src/app/api/admin/backups/__tests__/route.test.ts b/src/app/api/admin/backups/__tests__/route.test.ts index 70b583451..350c4756b 100644 --- a/src/app/api/admin/backups/__tests__/route.test.ts +++ b/src/app/api/admin/backups/__tests__/route.test.ts @@ -29,12 +29,23 @@ vi.mock("@/lib/db", () => ({ prisma: { $queryRaw: (...a: unknown[]) => queryRawMock(...a), user: { findMany: (...a: unknown[]) => findManyMock(...a) }, + offhostPurgeRequest: { + count: async () => 2, + findFirst: async () => ({ + requestedAt: new Date("2026-09-20T10:00:00.000Z"), + lastFailure: "AccessDenied", + }), + }, }, })); const configuredMock = vi.fn(() => true); vi.mock("@/lib/jobs/offhost-backup", () => ({ offhostBackupConfigured: () => configuredMock(), + probeOffhostLifecycle: async () => ({ + state: "missing", + expirationDays: null, + }), })); import { GET } from "../route"; @@ -139,10 +150,12 @@ describe("GET /api/admin/backups — off-host freshness", () => { const body = await read(); - expect(body.offhost).toEqual({ + expect(body.offhost).toMatchObject({ configured: false, periodHours: 24, rows: [], + // No bucket to ask on a host that is not configured. + lifecycle: { state: "unknown", expirationDays: null }, }); }); @@ -196,4 +209,20 @@ describe("GET /api/admin/backups — off-host freshness", () => { ]); expect(JSON.stringify(body.offhost)).not.toMatch(/bucket|endpoint|secret/i); }); + + it("carries the bucket's lifecycle verdict and the deletions it still owes", async () => { + configuredMock.mockReturnValue(true); + queryRawMock.mockResolvedValue([]); + findManyMock.mockResolvedValue([]); + const body = await read(); + expect(body.offhost.lifecycle).toEqual({ + state: "missing", + expirationDays: null, + }); + expect(body.offhost.pendingDeletions).toEqual({ + count: 2, + oldestRequestedAt: "2026-09-20T10:00:00.000Z", + lastFailure: "AccessDenied", + }); + }); }); diff --git a/src/app/api/admin/backups/route.ts b/src/app/api/admin/backups/route.ts index 09d9cc9f6..2a1939e08 100644 --- a/src/app/api/admin/backups/route.ts +++ b/src/app/api/admin/backups/route.ts @@ -27,7 +27,10 @@ import { annotate } from "@/lib/logging/context"; import { DATA_BACKUP_QUEUE } from "@/lib/jobs/data-backup-policy"; import { readLastQueueRun } from "@/lib/jobs/job-failures"; import { summariseBackupSchedule } from "@/lib/jobs/backup-schedule-status"; -import { offhostBackupConfigured } from "@/lib/jobs/offhost-backup"; +import { + offhostBackupConfigured, + probeOffhostLifecycle, +} from "@/lib/jobs/offhost-backup"; import { classifyOffhostBackup, OFFHOST_BACKUP_PERIOD_HOURS, @@ -164,6 +167,17 @@ export const GET = apiHandler(async () => { a.username.localeCompare(b.username), ); + const [purgeCount, oldestPurge, lifecycle] = await Promise.all([ + prisma.offhostPurgeRequest.count(), + prisma.offhostPurgeRequest.findFirst({ + orderBy: { requestedAt: "asc" }, + select: { requestedAt: true, lastFailure: true }, + }), + configured + ? probeOffhostLifecycle() + : Promise.resolve({ state: "unknown" as const, expirationDays: null }), + ]); + const payload: BackupsList = { rows: list, // Matches the retention window the backup-prune job enforces so the @@ -182,6 +196,12 @@ export const GET = apiHandler(async () => { // An unconfigured host has nothing to list, and the card says so rather // than painting an empty table that reads as "no problems". rows: configured ? offhostRows : [], + pendingDeletions: { + count: purgeCount, + oldestRequestedAt: oldestPurge?.requestedAt.toISOString() ?? null, + lastFailure: oldestPurge?.lastFailure ?? null, + }, + lifecycle, }, }; diff --git a/src/app/api/admin/backups/upload/route.ts b/src/app/api/admin/backups/upload/route.ts index b2b0e398f..26fea6e9e 100644 --- a/src/app/api/admin/backups/upload/route.ts +++ b/src/app/api/admin/backups/upload/route.ts @@ -29,7 +29,6 @@ * Phase B1 / criterion 2 of the v1.4.15 backup-completeness work. */ import { Readable } from "node:stream"; -import { createGunzip } from "node:zlib"; import { BodyTooLargeError, readBoundedBody } from "@/lib/labs/ocr-upload"; import { NextRequest } from "next/server"; @@ -44,6 +43,16 @@ import { import { apiError, apiSuccess, getClientIp } from "@/lib/api-response"; import { auditLog } from "@/lib/auth/audit"; import { BackupJsonError, scanBackupJson } from "@/lib/export/backup-json-scan"; +import { + BackupUploadDecodeError, + decodeBackupUpload, +} from "@/lib/export/backup-upload-decode"; +import { + assessBackupKeys, + BACKUP_KEY_MISSING_CODE, + BackupKeyIdCollector, + describeBackupKeyProblem, +} from "@/lib/export/backup-key-ids"; import { BACKUP_UPLOAD_TOO_LARGE_CODE, BackupBlobTooLargeError, @@ -101,50 +110,22 @@ class UploadRefused extends Error { } } -/** Bytes → bytes, gunzipped when the file starts with the gzip magic. */ +/** The file's bytes, gunzipped and counted on both ends. */ async function* decodedBytes( source: AsyncIterable, limit: number, ): AsyncGenerator { - let seen = 0; - async function* counted() { - for await (const chunk of source) { - seen += chunk.byteLength; - if (seen > limit) { - throw new UploadRefused( - 413, - `Upload exceeds ${Math.round(limit / 1024 / 1024)} MB limit`, - { reason: "file_size_exceeded", size: seen }, - ); - } - yield chunk; - } - } - const iterator = counted()[Symbol.asyncIterator](); - const first = await iterator.next(); - if (first.done) return; - const rest = (async function* () { - yield first.value; - for (;;) { - const next = await iterator.next(); - if (next.done) return; - yield next.value; - } - })(); - const head = first.value; - if (head.byteLength >= 2 && head[0] === 0x1f && head[1] === 0x8b) { - const gunzip = Readable.from(rest).pipe(createGunzip()); - try { - for await (const chunk of gunzip) yield chunk as Buffer; - } catch (err) { - if (err instanceof UploadRefused) throw err; - throw new UploadRefused(422, "Uploaded file is not valid gzip", { - reason: "invalid_gzip", + try { + yield* decodeBackupUpload(source, { compressedLimit: limit }); + } catch (err) { + if (err instanceof BackupUploadDecodeError) { + throw new UploadRefused(err.status, err.message, { + reason: err.reason, + ...(err.size !== undefined ? { size: err.size } : {}), }); } - return; + throw err; } - yield* rest; } export const POST = apiHandler(async (request: NextRequest) => { @@ -259,6 +240,7 @@ export const POST = apiHandler(async (request: NextRequest) => { } } let firstWithoutId: number | null = null; + const keys = new BackupKeyIdCollector(); let scanned; try { scanned = await scanBackupJson(storedAsRead(), { @@ -275,6 +257,7 @@ export const POST = apiHandler(async (request: NextRequest) => { if (!parsed.data.id && firstWithoutId === null) { firstWithoutId = index; } + keys.visit(element, "measurements"); }, }); } catch (err) { @@ -343,6 +326,27 @@ export const POST = apiHandler(async (request: NextRequest) => { ); } + // A file whose inner ciphertext needs a key this host does not have + // can be stored but never restored: the restore would refuse it, or + // worse, a restore on a host that had lost the key would write rows + // nobody can open. Refused here, where the operator is looking, with + // the key named. The instance settings are left to the restore, which + // checks them only when they are asked for back. + keys.visit(scanned.document); + const keyVerdict = assessBackupKeys(keys, { + ignoreSections: new Set(["appSettings"]), + }); + const keyProblem = describeBackupKeyProblem(keyVerdict); + if (keyProblem) { + const keyIds = [...keyVerdict.missing, ...keyVerdict.unreadable]; + throw new UploadRefused( + 422, + keyProblem, + { reason: "key_missing", keyIds }, + { errorCode: BACKUP_KEY_MISSING_CODE, keyIds }, + ); + } + // Make sure the userId on the file points at a user that exists in // this DB. Restore would fail later anyway, but failing here gives // the admin a precise error before any side-effect. diff --git a/src/app/api/admin/encryption/status/route.ts b/src/app/api/admin/encryption/status/route.ts index 76cc2b02f..4bda28af8 100644 --- a/src/app/api/admin/encryption/status/route.ts +++ b/src/app/api/admin/encryption/status/route.ts @@ -19,6 +19,10 @@ import { apiHandler, requireAdmin } from "@/lib/api-handler"; import { apiSuccess } from "@/lib/api-response"; import { annotate } from "@/lib/logging/context"; import { getConfiguredKeyIds } from "@/lib/crypto"; +import { + offhostBackupConfigured, + probeOffhostLifecycle, +} from "@/lib/jobs/offhost-backup"; import { scanCorpus, type CorpusClient, @@ -100,6 +104,100 @@ async function readRotationState(): Promise { }; } +/** + * Which keys the backups still need. + * + * Rotation re-encrypts every row in the database and re-seals every stored + * backup's envelope, but the content INSIDE a backup keeps the key it was + * written under: a restore writes it back verbatim. So "every row is on the + * active key" is not the same as "the old key can go". This answers the second + * question per key id, from what each copy recorded as it was written: the + * stored copies that need it and the oldest of them, and for the off-host + * bucket the last night an object needing it went in, plus how long the + * bucket's lifecycle rule keeps objects after that. + */ +export interface BackupKeyNeeds { + stored: Array<{ keyId: string; copies: number; oldestAt: string }>; + /** Copies written before v1.39.3, which recorded no key ids. */ + unrecorded: { copies: number; oldestAt: string | null }; + offhost: Array<{ + keyId: string; + firstWrittenAt: string; + lastWrittenAt: string; + /** When the last object needing this key expires; null when unknown. */ + neededUntil: string | null; + }>; + /** The bucket's expiry, when it has one the credential may read. */ + offhostExpirationDays: number | null; + /** Retired key ids some backup still needs (never the active one). */ + retiredKeysStillNeeded: string[]; +} + +async function readBackupKeyNeeds( + activeKeyId: string, + now: Date, +): Promise { + const [stored, unrecorded, offhostRows] = await Promise.all([ + prisma.$queryRaw< + Array<{ key_id: string; copies: number; oldest_at: Date }> + >` + SELECT k AS key_id, count(*)::int AS copies, min(created_at) AS oldest_at + FROM data_backups, unnest(inner_key_ids) AS k + WHERE inner_key_ids_recorded + GROUP BY k + ORDER BY k + `, + prisma.dataBackup.aggregate({ + where: { innerKeyIdsRecorded: false }, + _count: { _all: true }, + _min: { createdAt: true }, + }), + prisma.offhostBackupKeyUse.findMany({ orderBy: { keyId: "asc" } }), + ]); + const lifecycle = + offhostRows.length > 0 && offhostBackupConfigured() + ? await probeOffhostLifecycle() + : null; + const expirationDays = + lifecycle?.state === "configured" ? lifecycle.expirationDays : null; + + const offhost = offhostRows.map((row) => ({ + keyId: row.keyId, + firstWrittenAt: row.firstWrittenAt.toISOString(), + lastWrittenAt: row.lastWrittenAt.toISOString(), + neededUntil: + expirationDays === null + ? null + : new Date( + row.lastWrittenAt.getTime() + (expirationDays + 1) * 86_400_000, + ).toISOString(), + })); + + const needed = new Set(); + for (const row of stored) needed.add(row.key_id); + for (const row of offhost) { + if (row.neededUntil === null || Date.parse(row.neededUntil) > +now) { + needed.add(row.keyId); + } + } + needed.delete(activeKeyId); + + return { + stored: stored.map((row) => ({ + keyId: row.key_id, + copies: row.copies, + oldestAt: row.oldest_at.toISOString(), + })), + unrecorded: { + copies: unrecorded._count._all, + oldestAt: unrecorded._min.createdAt?.toISOString() ?? null, + }, + offhost, + offhostExpirationDays: expirationDays, + retiredKeysStillNeeded: [...needed].sort(), + }; +} + export const GET = apiHandler(async () => { await requireAdmin(); annotate({ action: { name: "admin.encryption.status" } }); @@ -114,6 +212,7 @@ export const GET = apiHandler(async () => { } const rotation = await readRotationState(); + const backups = await readBackupKeyNeeds(scan.activeKeyId, new Date()); annotate({ meta: { @@ -133,5 +232,14 @@ export const GET = apiHandler(async () => { staleRows: scan.staleRows, columns: scan.columns, rotation, + backups, + // Every row rotated AND no backup still needs a retired key: the one + // state in which dropping an old key loses nothing. Copies that recorded + // no key ids count against it, because they may hold any key that + // existed when they were written. + safeToDropRetiredKeys: + scan.rotationComplete && + backups.retiredKeysStillNeeded.length === 0 && + backups.unrecorded.copies === 0, }); }); diff --git a/src/app/api/admin/import-apple-health-export/__tests__/route.test.ts b/src/app/api/admin/import-apple-health-export/__tests__/route.test.ts index 952f2f006..fcb453ed6 100644 --- a/src/app/api/admin/import-apple-health-export/__tests__/route.test.ts +++ b/src/app/api/admin/import-apple-health-export/__tests__/route.test.ts @@ -25,6 +25,7 @@ vi.mock("next/headers", () => ({ })); const mocks = vi.hoisted(() => ({ + runningImportFindFirst: vi.fn(async () => null), checkRateLimit: vi.fn().mockResolvedValue({ allowed: true }), bossSend: vi.fn().mockResolvedValue("boss-job-1"), getGlobalBoss: vi.fn(), @@ -60,6 +61,16 @@ vi.mock("@/lib/db", () => ({ findFirst: mocks.importJobFindFirst, update: mocks.importJobUpdate, }, + // The one-running-import check and the new row, in one transaction. + // The check finds nothing running unless a test says otherwise. + $transaction: async (fn: (tx: unknown) => unknown) => + fn({ + $executeRaw: async () => 0, + importJob: { + findFirst: mocks.runningImportFindFirst, + create: mocks.importJobCreate, + }, + }), user: { findUnique: mocks.userFindUnique, }, diff --git a/src/app/api/admin/import-apple-health-export/route.ts b/src/app/api/admin/import-apple-health-export/route.ts index 67e8ecd32..b3ea40528 100644 --- a/src/app/api/admin/import-apple-health-export/route.ts +++ b/src/app/api/admin/import-apple-health-export/route.ts @@ -27,6 +27,11 @@ import { type AppleHealthImportPayload, } from "@/lib/jobs/apple-health-import-worker"; import { streamMultipartToDisk } from "@/lib/multipart/stream-to-disk"; +import { + createImportJobUnlessBusy, + discardStagedUpload, + IMPORT_BUSY_CODE, +} from "@/lib/import/apple-health-staging"; import { unlink } from "node:fs/promises"; export const dynamic = "force-dynamic"; @@ -96,6 +101,7 @@ export const POST = apiHandler(async (request: NextRequest) => { ipAddress: getClientIp(request), details: { reason: "missing_user_id" }, }); + await discardStagedUpload(uploaded.filePath); return apiError("Multipart field 'userId' is required", 422); } @@ -109,6 +115,7 @@ export const POST = apiHandler(async (request: NextRequest) => { ipAddress: getClientIp(request), details: { reason: "target_user_missing", targetUserId }, }); + await discardStagedUpload(uploaded.filePath); return apiError(`Target user '${targetUserId}' does not exist`, 422); } @@ -149,19 +156,27 @@ export const POST = apiHandler(async (request: NextRequest) => { ipAddress: getClientIp(request), details: { reason: "worker_not_running" }, }); + await discardStagedUpload(uploaded.filePath); throw new HttpError(503, "Background worker is not running"); } - const importJob = await prisma.importJob.create({ - data: { - userId: targetUser.id, - triggeredByAdminId: admin.id, - status: "queued", - uploadBytes: uploaded.bytes, - uploadSha256: uploaded.sha256, - parserRevision: APPLE_HEALTH_IMPORT_PARSER_REVISION, - }, + const staged = await createImportJobUnlessBusy(prisma, { + userId: targetUser.id, + triggeredByAdminId: admin.id, + status: "queued", + uploadBytes: uploaded.bytes, + uploadSha256: uploaded.sha256, + parserRevision: APPLE_HEALTH_IMPORT_PARSER_REVISION, }); + if ("busy" in staged) { + await discardStagedUpload(uploaded.filePath); + return apiError( + "An Apple Health import is already running for this account. Wait for it to finish, then upload again.", + 409, + { errorCode: IMPORT_BUSY_CODE, jobId: staged.busy.id }, + ); + } + const importJob = staged.created; const payload: AppleHealthImportPayload = { userId: targetUser.id, @@ -178,6 +193,19 @@ export const POST = apiHandler(async (request: NextRequest) => { payload, APPLE_HEALTH_IMPORT_SEND_OPTIONS, ); + if (!bossJobId) { + await prisma.importJob.update({ + where: { id: importJob.id }, + data: { + status: "failed", + failureReason: + "The import could not be queued. Upload the export again.", + completedAt: new Date(), + }, + }); + await discardStagedUpload(uploaded.filePath); + throw new HttpError(503, "The import could not be queued"); + } await prisma.importJob.update({ where: { id: importJob.id }, diff --git a/src/app/api/import/apple-health-export/__tests__/route.test.ts b/src/app/api/import/apple-health-export/__tests__/route.test.ts index 5a0aad7ef..8acfc3b04 100644 --- a/src/app/api/import/apple-health-export/__tests__/route.test.ts +++ b/src/app/api/import/apple-health-export/__tests__/route.test.ts @@ -27,6 +27,7 @@ vi.mock("next/headers", () => ({ })); const mocks = vi.hoisted(() => ({ + runningImportFindFirst: vi.fn(async () => null), checkRateLimit: vi.fn().mockResolvedValue({ allowed: true }), bossSend: vi.fn().mockResolvedValue("boss-job-1"), getGlobalBoss: vi.fn(), @@ -35,6 +36,7 @@ const mocks = vi.hoisted(() => ({ importJobUpdate: vi.fn(), streamToDisk: vi.fn(), unlink: vi.fn().mockResolvedValue(undefined), + rm: vi.fn().mockResolvedValue(undefined), })); vi.mock("@/lib/rate-limit", () => ({ @@ -51,7 +53,7 @@ vi.mock("@/lib/multipart/stream-to-disk", () => ({ vi.mock("node:fs/promises", async (importOriginal) => { const actual = await importOriginal(); - return { ...actual, unlink: mocks.unlink }; + return { ...actual, unlink: mocks.unlink, rm: mocks.rm }; }); vi.mock("@/lib/db", () => ({ @@ -61,6 +63,16 @@ vi.mock("@/lib/db", () => ({ findFirst: mocks.importJobFindFirst, update: mocks.importJobUpdate, }, + // The one-running-import check and the new row, in one transaction. + // The check finds nothing running unless a test says otherwise. + $transaction: async (fn: (tx: unknown) => unknown) => + fn({ + $executeRaw: async () => 0, + importJob: { + findFirst: mocks.runningImportFindFirst, + create: mocks.importJobCreate, + }, + }), }, })); @@ -90,6 +102,7 @@ beforeEach(() => { textFields: {}, }); mocks.unlink.mockResolvedValue(undefined); + mocks.rm.mockResolvedValue(undefined); }); function multipartReq(): NextRequest { @@ -220,3 +233,51 @@ describe("POST /api/import/apple-health-export — dedup by content hash (issue ); }); }); + +// The staged upload is the person's export in plain text. Every exit that +// leaves no worker to consume it removes it, and a second import for the same +// account is refused while one is running. +describe("POST /api/import/apple-health-export — the staged upload on every exit", () => { + const removed = () => + mocks.rm.mock.calls.some(([path]) => path === STAGED_PATH); + + it("refuses a second import while one is running, and removes the upload", async () => { + vi.mocked(getSession).mockResolvedValue(SESSION_OK as never); + mocks.runningImportFindFirst.mockResolvedValue({ + id: "ij-running", + status: "parsing", + } as never); + const res = await POST(multipartReq()); + expect(res.status).toBe(409); + const body = await res.json(); + expect(body.meta).toMatchObject({ + errorCode: "import.apple_health.busy", + jobId: "ij-running", + }); + expect(mocks.importJobCreate).not.toHaveBeenCalled(); + expect(mocks.bossSend).not.toHaveBeenCalled(); + expect(removed()).toBe(true); + }); + + it("removes the upload when no worker is running", async () => { + vi.mocked(getSession).mockResolvedValue(SESSION_OK as never); + mocks.getGlobalBoss.mockReturnValue(null); + const res = await POST(multipartReq()); + expect(res.status).toBe(503); + expect(removed()).toBe(true); + }); + + it("fails the row and removes the upload when the queue takes no job", async () => { + vi.mocked(getSession).mockResolvedValue(SESSION_OK as never); + mocks.bossSend.mockResolvedValue(null); + const res = await POST(multipartReq()); + expect(res.status).toBe(503); + expect(mocks.importJobUpdate).toHaveBeenCalledWith( + expect.objectContaining({ + where: { id: "ij-1" }, + data: expect.objectContaining({ status: "failed" }), + }), + ); + expect(removed()).toBe(true); + }); +}); diff --git a/src/app/api/import/apple-health-export/route.ts b/src/app/api/import/apple-health-export/route.ts index 131fd2b68..6919703ff 100644 --- a/src/app/api/import/apple-health-export/route.ts +++ b/src/app/api/import/apple-health-export/route.ts @@ -35,6 +35,11 @@ import { type AppleHealthImportPayload, } from "@/lib/jobs/apple-health-import-worker"; import { streamMultipartToDisk } from "@/lib/multipart/stream-to-disk"; +import { + createImportJobUnlessBusy, + discardStagedUpload, + IMPORT_BUSY_CODE, +} from "@/lib/import/apple-health-staging"; import { unlink } from "node:fs/promises"; export const dynamic = "force-dynamic"; @@ -147,18 +152,28 @@ export const POST = apiHandler(async (request: NextRequest) => { ipAddress: getClientIp(request), details: { reason: "worker_not_running" }, }); + // Nothing will ever read the staged export; it must not stay in /tmp. + await discardStagedUpload(uploaded.filePath); throw new HttpError(503, "Background worker is not running"); } - const importJob = await prisma.importJob.create({ - data: { - userId: user.id, - status: "queued", - uploadBytes: uploaded.bytes, - uploadSha256: uploaded.sha256, - parserRevision: APPLE_HEALTH_IMPORT_PARSER_REVISION, - }, + // One running import per account (see `apple-health-staging.ts`). + const staged = await createImportJobUnlessBusy(prisma, { + userId: user.id, + status: "queued", + uploadBytes: uploaded.bytes, + uploadSha256: uploaded.sha256, + parserRevision: APPLE_HEALTH_IMPORT_PARSER_REVISION, }); + if ("busy" in staged) { + await discardStagedUpload(uploaded.filePath); + return apiError( + "An Apple Health import is already running for this account. Wait for it to finish, then upload again.", + 409, + { errorCode: IMPORT_BUSY_CODE, jobId: staged.busy.id }, + ); + } + const importJob = staged.created; const payload: AppleHealthImportPayload = { userId: user.id, @@ -174,6 +189,21 @@ export const POST = apiHandler(async (request: NextRequest) => { payload, APPLE_HEALTH_IMPORT_SEND_OPTIONS, ); + if (!bossJobId) { + // The queue did not take the job, so no worker will consume the upload + // or move the row on. Fail the row now and remove the export. + await prisma.importJob.update({ + where: { id: importJob.id }, + data: { + status: "failed", + failureReason: + "The import could not be queued. Upload the export again.", + completedAt: new Date(), + }, + }); + await discardStagedUpload(uploaded.filePath); + throw new HttpError(503, "The import could not be queued"); + } await prisma.importJob.update({ where: { id: importJob.id }, diff --git a/src/app/api/settings/account/route.ts b/src/app/api/settings/account/route.ts index 4f863defc..60e06374a 100644 --- a/src/app/api/settings/account/route.ts +++ b/src/app/api/settings/account/route.ts @@ -8,6 +8,10 @@ import { MFA_STEP_UP_MAX_AGE_SECONDS, } from "@/lib/api-handler"; import { annotate } from "@/lib/logging/context"; +import { + kickOffhostPurge, + requestOffhostPurge, +} from "@/lib/jobs/offhost-purge"; import { deleteGuardianAccountWithLifecycle, LastManagedGuardianError, @@ -101,6 +105,10 @@ export const DELETE = apiHandler(async (request: NextRequest) => { // genuinely complete erasure. await tx.auditLog.deleteMany({ where: { userId } }); + // The nightly off-host copies go with the account. The request commits + // with the deletion; the purge job deletes the objects. + await requestOffhostPurge(tx, userId, "account_deleted"); + // Delete user — all other related data is removed via onDelete: Cascade. await tx.user.delete({ where: { id: userId } }); }); @@ -115,6 +123,8 @@ export const DELETE = apiHandler(async (request: NextRequest) => { throw error; } + await kickOffhostPurge(); + annotate({ action: { name: "settings.account.delete" }, meta: { username }, diff --git a/src/app/api/settings/data/route.ts b/src/app/api/settings/data/route.ts index 409320ccc..2c3187e0f 100644 --- a/src/app/api/settings/data/route.ts +++ b/src/app/api/settings/data/route.ts @@ -1,5 +1,9 @@ import { prisma } from "@/lib/db"; import { auditLog } from "@/lib/auth/audit"; +import { + kickOffhostPurge, + requestOffhostPurge, +} from "@/lib/jobs/offhost-purge"; import { apiSuccess, apiError, getClientIp } from "@/lib/api-response"; import { NextRequest } from "next/server"; import { @@ -101,6 +105,11 @@ export const DELETE = apiHandler(async (request: NextRequest) => { // does not. Same classification contract as the model list. await tx.user.update({ where: { id: userId }, data: USER_RESET }); + // The nightly off-host copies of the record are part of "all your + // data". The request commits with the wipe; the purge job deletes + // the objects and removes it. + await requestOffhostPurge(tx, userId, "data_wiped"); + return perModel; }, { @@ -144,6 +153,7 @@ export const DELETE = apiHandler(async (request: NextRequest) => { // v1.16.9 — every cached payload was built on the rows just deleted; // hard-evict the user's buckets so no surface serves pre-wipe data. invalidateUserData(userId); + await kickOffhostPurge(); return apiSuccess({ cleared: true, deletedRows, models: counts }); }); diff --git a/src/components/admin/__tests__/offhost-backups-section.test.tsx b/src/components/admin/__tests__/offhost-backups-section.test.tsx index d3dd46985..1bc8a886d 100644 --- a/src/components/admin/__tests__/offhost-backups-section.test.tsx +++ b/src/components/admin/__tests__/offhost-backups-section.test.tsx @@ -23,6 +23,11 @@ vi.mock("@tanstack/react-query", () => ({ import { I18nProvider } from "@/lib/i18n/context"; import { OffhostBackupsSection } from "../offhost-backups-section"; +const QUIET = { + pendingDeletions: { count: 0, oldestRequestedAt: null, lastFailure: null }, + lifecycle: { state: "configured" as const, expirationDays: 30 }, +}; + function render(data: BackupsList["offhost"] | null): string { queryResult.current = { data: @@ -46,7 +51,12 @@ function render(data: BackupsList["offhost"] | null): string { describe("", () => { it("says off-host backup is not configured rather than showing an empty list", () => { - const html = render({ configured: false, periodHours: 24, rows: [] }); + const html = render({ + configured: false, + periodHours: 24, + rows: [], + ...QUIET, + }); expect(html).toContain("Off-host backup is not configured"); expect(html).not.toContain('data-slot="offhost-backup-rows"'); }); @@ -55,6 +65,7 @@ describe("", () => { const html = render({ configured: true, periodHours: 24, + ...QUIET, rows: [ { userId: "u1", @@ -109,4 +120,33 @@ describe("", () => { // Never the bucket's own coordinates. expect(html).not.toMatch(/secret|access[- ]key/i); }); + + it("says when the bucket keeps copies forever, and when deletions are stuck", () => { + const html = render({ + configured: true, + periodHours: 24, + rows: [], + pendingDeletions: { + count: 2, + oldestRequestedAt: "2026-09-20T10:00:00.000Z", + lastFailure: "AccessDenied", + }, + lifecycle: { state: "missing", expirationDays: null }, + }); + expect(html).toContain('data-lifecycle-state="missing"'); + expect(html).toContain("no lifecycle rule"); + expect(html).toContain('data-slot="offhost-pending-deletions"'); + expect(html).toContain("AccessDenied"); + }); + + it("states the expiry the bucket enforces", () => { + const html = render({ + configured: true, + periodHours: 24, + rows: [], + ...QUIET, + }); + expect(html).toContain("after 30 days"); + expect(html).not.toContain('data-slot="offhost-pending-deletions"'); + }); }); diff --git a/src/components/admin/backup-restore-status.tsx b/src/components/admin/backup-restore-status.tsx index 7707f7aa4..3b9fed7c7 100644 --- a/src/components/admin/backup-restore-status.tsx +++ b/src/components/admin/backup-restore-status.tsx @@ -71,6 +71,7 @@ function restoredCount(job: BackupRestoreJobView): number { function failureText( code: BackupRestoreFailureCode, t: ReturnType["t"], + keyIds: readonly string[] = [], ): string { switch (code) { case "backup_not_found": @@ -91,6 +92,12 @@ function failureText( return t("admin.section.backups.restoreFailureSectionMissing"); case "document_ciphertext_missing": return t("admin.section.backups.restoreFailureDocument"); + case "backup.key.missing": + return t("admin.section.backups.restoreFailureKeyMissing", { + keys: keyIds.join(", "), + }); + case "backup.foreign_reference": + return t("admin.section.backups.restoreFailureForeignReference"); case "time_budget": return t("admin.section.backups.restoreFailureTimeBudget"); case "transaction_failed": @@ -289,7 +296,9 @@ export function RestoreJobStatus({

{t("admin.section.backups.restoreJobFailedTitle", { username })}

-

{failureText(code, t)}

+

+ {failureText(code, t, job.failure?.keyIds)} +

{changedNothing ? (

{t("admin.section.backups.restoreJobFailedKept")} diff --git a/src/components/admin/backups-section.tsx b/src/components/admin/backups-section.tsx index c7f9bd7c5..53d708383 100644 --- a/src/components/admin/backups-section.tsx +++ b/src/components/admin/backups-section.tsx @@ -238,11 +238,23 @@ function RestoreRowDialog({ {t("admin.section.backups.previewLoading")}

)} - {preview.isError && ( -

- {t("admin.section.backups.previewUnavailable")} -

- )} + {preview.isError && + (preview.error instanceof ApiError && + preview.error.meta?.errorCode === "backup.key.missing" ? ( + // The one preview failure the restore would repeat: say which + // key, so the operator can put it back before trying. +

+ {t("admin.section.backups.previewKeyMissing", { + keys: Array.isArray(preview.error.meta.keyIds) + ? preview.error.meta.keyIds.map(String).join(", ") + : "", + })} +

+ ) : ( +

+ {t("admin.section.backups.previewUnavailable")} +

+ ))} {preview.isSuccess && (
    {previewRows.map((entry) => ( diff --git a/src/components/admin/encryption-section.tsx b/src/components/admin/encryption-section.tsx index 7b043fd78..dda5efb26 100644 --- a/src/components/admin/encryption-section.tsx +++ b/src/components/admin/encryption-section.tsx @@ -13,7 +13,13 @@ */ import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query"; -import { KeyRound, Loader2, RotateCw, ShieldCheck } from "lucide-react"; +import { + Archive, + KeyRound, + Loader2, + RotateCw, + ShieldCheck, +} from "lucide-react"; import { toast } from "sonner"; import { AlertDialog, @@ -32,7 +38,7 @@ import { StatTile } from "@/components/admin/_stat-tile"; import { Button } from "@/components/ui/button"; import { SettingsCard } from "@/components/settings/settings-card"; import { SettingsCardHeader } from "@/components/settings/_card-header"; -import { useTranslations } from "@/lib/i18n/context"; +import { useFormatters, useTranslations } from "@/lib/i18n/context"; import { queryKeys } from "@/lib/query-keys"; import { apiFetchRaw, apiGet } from "@/lib/api/api-fetch"; import { getApiErrorMessage } from "./_shared"; @@ -60,10 +66,25 @@ interface EncryptionStatus { lastCompletedAt: string | null; lastResult: { scanned: number; rotated: number; errors: number } | null; }; + /** Which keys the backups still need; see the status route. */ + backups: { + stored: Array<{ keyId: string; copies: number; oldestAt: string }>; + unrecorded: { copies: number; oldestAt: string | null }; + offhost: Array<{ + keyId: string; + firstWrittenAt: string; + lastWrittenAt: string; + neededUntil: string | null; + }>; + offhostExpirationDays: number | null; + retiredKeysStillNeeded: string[]; + }; + safeToDropRetiredKeys: boolean; } export function EncryptionSection() { const { t } = useTranslations(); + const fmt = useFormatters(); const queryClient = useQueryClient(); const statusQuery = useQuery({ @@ -166,10 +187,20 @@ export function EncryptionSection() { value={s.staleRows.toLocaleString()} /> - {s.rotationComplete ? ( + {s.safeToDropRetiredKeys ? ( {t("admin.section.encryption.safeToDropLegacy")} + ) : s.rotationComplete ? ( + // Every row is on the active key, and that is not the whole + // answer: a backup keeps the key its content was written under. + + {s.backups.retiredKeysStillNeeded.length > 0 + ? t("admin.section.encryption.backupsStillNeed", { + keys: s.backups.retiredKeysStillNeeded.join(", "), + }) + : t("admin.section.encryption.backupsUnrecordedBadge")} + ) : ( {t("admin.section.encryption.rotationIncomplete", { @@ -179,6 +210,56 @@ export function EncryptionSection() { )} + {/* ── Keys the backups still need ───────────────────────────── */} + + +

    {t("admin.section.encryption.backupsDetail")}

    +
      + {s.backups.stored.map((row) => ( +
    • + {t("admin.section.encryption.backupsStoredLine", { + key: row.keyId, + count: row.copies, + when: fmt.dateTime(row.oldestAt), + })} +
      • + ))} + {s.backups.unrecorded.copies > 0 && s.backups.unrecorded.oldestAt ? ( +
    • + {t("admin.section.encryption.backupsUnrecordedLine", { + count: s.backups.unrecorded.copies, + when: fmt.dateTime(s.backups.unrecorded.oldestAt), + })} +
      • + ) : null} + {s.backups.offhost.map((row) => ( +
    • + {row.neededUntil + ? t("admin.section.encryption.backupsOffhostLine", { + key: row.keyId, + when: fmt.dateTime(row.lastWrittenAt), + until: fmt.dateTime(row.neededUntil), + }) + : t("admin.section.encryption.backupsOffhostLineUnknown", { + key: row.keyId, + when: fmt.dateTime(row.lastWrittenAt), + })} +
      • + ))} + {s.backups.stored.length === 0 && + s.backups.unrecorded.copies === 0 && + s.backups.offhost.length === 0 ? ( +
    • + {t("admin.section.encryption.backupsNone")} +
      • + ) : null} +
    +     + {/* ── Rotation status + trigger ──────────────────────────────── */} +

    + {offhost.lifecycle.state === "configured" + ? t("admin.section.backups.offhost.lifecycleConfigured", { + days: offhost.lifecycle.expirationDays ?? 0, + }) + : offhost.lifecycle.state === "missing" + ? t("admin.section.backups.offhost.lifecycleMissing") + : t("admin.section.backups.offhost.lifecycleUnknown")} +

    + {offhost.pendingDeletions.count > 0 ? ( +

    + {offhost.pendingDeletions.lastFailure + ? t("admin.section.backups.offhost.pendingDeletionsFailing", { + count: offhost.pendingDeletions.count, + error: offhost.pendingDeletions.lastFailure, + }) + : t("admin.section.backups.offhost.pendingDeletions", { + count: offhost.pendingDeletions.count, + })} +

    + ) : null} ) : null}     diff --git a/src/lib/crypto.ts b/src/lib/crypto.ts index 39a6583d7..d46a06fe2 100644 --- a/src/lib/crypto.ts +++ b/src/lib/crypto.ts @@ -465,13 +465,21 @@ export interface RawStreamEncryptor { final(): Buffer; } -/** Open a byte-level streaming encryptor under an explicit 32-byte key. */ -export function createRawStreamEncryptor(key: Buffer): RawStreamEncryptor { +/** + * Open a byte-level streaming encryptor under an explicit 32-byte key. + * `aad`, when given, is authenticated with the ciphertext and has to be given + * again, byte for byte, to decrypt it. + */ +export function createRawStreamEncryptor( + key: Buffer, + aad?: Buffer, +): RawStreamEncryptor { if (key.byteLength !== 32) { throw new Error("Stream encryption key must be 32 bytes"); } const iv = randomBytes(IV_LENGTH); const cipher = createCipheriv(ALGORITHM, key, iv); + if (aad) cipher.setAAD(aad); let done = false; return { @@ -499,7 +507,11 @@ export function createRawStreamEncryptor(key: Buffer): RawStreamEncryptor { * unverified plaintext to save a copy would trade the authentication for * memory. */ -export function decryptRawStream(packed: Buffer, key: Buffer): Buffer { +export function decryptRawStream( + packed: Buffer, + key: Buffer, + aad?: Buffer, +): Buffer { if (packed.byteLength < IV_LENGTH + AUTH_TAG_LENGTH) { throw new Error("Streamed ciphertext is truncated"); } @@ -508,6 +520,7 @@ export function decryptRawStream(packed: Buffer, key: Buffer): Buffer { const ct = packed.subarray(IV_LENGTH, packed.byteLength - AUTH_TAG_LENGTH); const dec = createDecipheriv(ALGORITHM, key, iv); dec.setAuthTag(tag); + if (aad) dec.setAAD(aad); return Buffer.concat([dec.update(ct), dec.final()]); } diff --git a/src/lib/data-wipe/wipe-plan.ts b/src/lib/data-wipe/wipe-plan.ts index 1e4f2ff9e..3e91d5379 100644 --- a/src/lib/data-wipe/wipe-plan.ts +++ b/src/lib/data-wipe/wipe-plan.ts @@ -227,9 +227,9 @@ export const WIPE_MODELS = [ // ended. About this account's record, so they go with it. "BackupRestoreJob", // What this host last put in the operator's bucket for this account. The - // objects themselves stay — the worker holds no DeleteObject grant and the - // bucket's lifecycle rule owns their retirement — but the row saying when - // they were written is about this account and goes with it. + // objects themselves are deleted by the purge request the wipe writes + // (`requestOffhostPurge`); this row, saying when they were written, is about + // this account and goes with it. "OffhostBackupState", "AuditLog", "ConsentReceipt", @@ -297,6 +297,10 @@ export const INSTANCE_SCOPED: Readonly> = { AppSettings: "instance-wide configuration singleton", RateLimit: "instance-wide rate-limit buckets, keyed by string, self-expiring", HostMetric: "instance-wide host telemetry, not attributable to an account", + OffhostBackupKeyUse: + "per encryption key, when the nightly off-host run last wrote an object needing it; says nothing about any account", + OffhostPurgeRequest: + "the deletions an account deletion or wipe asked of the off-host bucket; written by the wipe itself and removed once the bucket holds nothing of the account", CycleSymptomCategory: "seeded catalogue shared by every account; a user's custom symptoms live on CycleSymptom", IllnessSymptom: "seeded symptom catalogue shared by every account", diff --git a/src/lib/export/__tests__/backup-key-ids.test.ts b/src/lib/export/__tests__/backup-key-ids.test.ts new file mode 100644 index 000000000..03377c809 --- /dev/null +++ b/src/lib/export/__tests__/backup-key-ids.test.ts @@ -0,0 +1,178 @@ +/** + * The key ids a backup's inner ciphertext needs, and the verdict against the + * keys this host has. + * + * The values are produced by the real codecs under real keys, then the host's + * key set is changed underneath them, which is exactly what an operator does + * when a rotation is followed by dropping the old key. + */ +import { Buffer } from "node:buffer"; + +import { afterEach, beforeEach, describe, expect, it } from "vitest"; + +import { _resetCryptoCacheForTests, encrypt, encryptBytes } from "@/lib/crypto"; +import { + assessBackupKeys, + BackupKeyIdCollector, + createBackupKeyIdTextScanner, + describeBackupKeyProblem, + innerCiphertextKeyId, +} from "../backup-key-ids"; + +const K1 = "11".repeat(32); +const K2 = "22".repeat(32); +const K3 = "33".repeat(32); + +function useKeys(keys: Record, active: string) { + delete process.env.ENCRYPTION_KEY; + process.env.ENCRYPTION_KEYS = JSON.stringify(keys); + process.env.ENCRYPTION_ACTIVE_KEY_ID = active; + _resetCryptoCacheForTests(); +} + +const saved = { + key: process.env.ENCRYPTION_KEY, + keys: process.env.ENCRYPTION_KEYS, + active: process.env.ENCRYPTION_ACTIVE_KEY_ID, +}; + +beforeEach(() => useKeys({ old: K1, cur: K2 }, "old")); +afterEach(() => { + for (const [name, value] of [ + ["ENCRYPTION_KEY", saved.key], + ["ENCRYPTION_KEYS", saved.keys], + ["ENCRYPTION_ACTIVE_KEY_ID", saved.active], + ] as const) { + if (value === undefined) delete process.env[name]; + else process.env[name] = value; + } + _resetCryptoCacheForTests(); +}); + +/** A Bytes column holding the string codec, as a DR payload carries it. */ +const bytesString = (plain: string) => + Buffer.from(encrypt(plain), "utf8").toString("base64"); +/** A Bytes column holding the binary codec, as a document's content. */ +const binary = (plain: string) => + encryptBytes(Buffer.from(plain)).toString("base64"); + +describe("innerCiphertextKeyId", () => { + it("reads the key id of each stored layout", () => { + expect(innerCiphertextKeyId(encrypt("note"))).toEqual({ + keyId: "old", + form: "string", + }); + expect(innerCiphertextKeyId(bytesString("note"))).toEqual({ + keyId: "old", + form: "bytes-string", + }); + expect(innerCiphertextKeyId(binary("document"))).toEqual({ + keyId: "old", + form: "binary", + }); + }); + + it("reads a value with no key id as the legacy v1 key", () => { + const legacy = Buffer.from("A".repeat(40)).toString("base64"); + expect(innerCiphertextKeyId(legacy)?.keyId).toBe("v1"); + }); + + it("ignores an empty value, a short one, and one that is not base64", () => { + expect(innerCiphertextKeyId("")).toBeNull(); + expect(innerCiphertextKeyId("yes")).toBeNull(); + expect( + innerCiphertextKeyId("not ciphertext at all, and long enough to count!"), + ).toBeNull(); + }); +}); + +describe("assessBackupKeys", () => { + function fileWrittenUnderOld() { + return { + userId: "u1", + moodEntries: [{ noteEncrypted: bytesString("a mood note") }], + documents: [{ contentEncrypted: binary("%PDF-1.4 …") }], + appSettings: { adminAiKeyEncrypted: encrypt("sk-something") }, + }; + } + + it("passes a file whose keys are all configured and open", () => { + const collector = new BackupKeyIdCollector(); + collector.visit(fileWrittenUnderOld()); + collector.visit({ notesEncrypted: bytesString("x") }, "measurements"); + const verdict = assessBackupKeys(collector); + expect(verdict).toEqual({ + keyIds: ["old"], + missing: [], + unreadable: [], + affectedValues: 0, + }); + expect(describeBackupKeyProblem(verdict)).toBeNull(); + }); + + it("names a key the file needs and the host no longer has", () => { + const file = fileWrittenUnderOld(); + useKeys({ cur: K2 }, "cur"); + const collector = new BackupKeyIdCollector(); + collector.visit(file); + const verdict = assessBackupKeys(collector); + expect(verdict.missing).toEqual(["old"]); + expect(verdict.affectedValues).toBe(3); + expect(describeBackupKeyProblem(verdict)).toContain("'old'"); + }); + + it("leaves out sections the caller will not write", () => { + useKeys({ old: K1, cur: K2 }, "cur"); + const file = { + moodEntries: [{ noteEncrypted: bytesString("fine") }], + appSettings: { + adminAiKeyEncrypted: `gone.${Buffer.alloc(40).toString("base64")}`, + }, + }; + const collector = new BackupKeyIdCollector(); + collector.visit(file); + expect(assessBackupKeys(collector).missing).toEqual(["gone"]); + expect( + assessBackupKeys(collector, { ignoreSections: new Set(["appSettings"]) }) + .missing, + ).toEqual([]); + }); + + it("catches an id that is configured with different key material", () => { + const file = fileWrittenUnderOld(); + useKeys({ old: K3, cur: K2 }, "cur"); + const collector = new BackupKeyIdCollector(); + collector.visit(file); + const verdict = assessBackupKeys(collector); + expect(verdict.missing).toEqual([]); + expect(verdict.unreadable).toEqual(["old"]); + }); +}); + +describe("createBackupKeyIdTextScanner", () => { + it("finds every key id in JSON fed in arbitrary pieces", () => { + const early = bytesString("written before the rotation"); + useKeys({ old: K1, cur: K2 }, "cur"); + const late = binary("written after it"); + const json = JSON.stringify({ + measurements: [{ id: "m1", notesEncrypted: early }], + documents: [{ contentEncrypted: late, note: "plain" }], + plain: "noteEncrypted", + }); + for (const size of [1, 7, 33, 4096]) { + const scanner = createBackupKeyIdTextScanner(); + for (let i = 0; i < json.length; i += size) { + scanner.feed(Buffer.from(json.slice(i, i + size), "utf8")); + } + expect(scanner.keyIds()).toEqual(["cur", "old"]); + } + }); + + it("reads pretty-printed JSON", () => { + const scanner = createBackupKeyIdTextScanner(); + scanner.feed( + JSON.stringify({ a: { noteEncrypted: bytesString("x") } }, null, 2), + ); + expect(scanner.keyIds()).toEqual(["old"]); + }); +}); diff --git a/src/lib/export/__tests__/backup-upload-decode.test.ts b/src/lib/export/__tests__/backup-upload-decode.test.ts new file mode 100644 index 000000000..ec41aae02 --- /dev/null +++ b/src/lib/export/__tests__/backup-upload-decode.test.ts @@ -0,0 +1,70 @@ +/** + * An uploaded backup is counted after the gunzip as well as before it. + * Mutation that must turn this red: drop the inflated count (the bomb reads + * through to the end). + */ +import { gzipSync } from "node:zlib"; + +import { describe, expect, it } from "vitest"; + +import { + BackupUploadDecodeError, + decodeBackupUpload, +} from "../backup-upload-decode"; + +async function drain( + bytes: Buffer, + limits: Parameters[1], +): Promise { + async function* source() { + for (let i = 0; i < bytes.length; i += 4096) { + yield bytes.subarray(i, i + 4096); + } + } + const out: Buffer[] = []; + for await (const chunk of decodeBackupUpload(source(), limits)) { + out.push(Buffer.from(chunk)); + } + return Buffer.concat(out); +} + +describe("decodeBackupUpload", () => { + it("passes plain JSON through and gunzips a gzip file", async () => { + const json = Buffer.from(JSON.stringify({ userId: "u1", n: [1, 2, 3] })); + expect(await drain(json, { compressedLimit: 1024 })).toEqual(json); + expect(await drain(gzipSync(json), { compressedLimit: 1024 })).toEqual( + json, + ); + }); + + it("refuses a gzip that inflates far past its own size", async () => { + const bomb = gzipSync(Buffer.alloc(32 * 1024 * 1024)); + expect(bomb.length).toBeLessThan(64 * 1024); + const refused = drain(bomb, { + compressedLimit: 1024 * 1024, + minAllowance: 1024 * 1024, + ratio: 100, + }); + await expect(refused).rejects.toBeInstanceOf(BackupUploadDecodeError); + await expect(refused).rejects.toMatchObject({ + status: 413, + reason: "inflated_size_exceeded", + }); + }); + + it("refuses a file that is not valid gzip", async () => { + const broken = Buffer.concat([ + Buffer.from([0x1f, 0x8b]), + Buffer.from("not gzip"), + ]); + await expect( + drain(broken, { compressedLimit: 1024 }), + ).rejects.toMatchObject({ status: 422, reason: "invalid_gzip" }); + }); + + it("refuses more bytes than the upload limit", async () => { + await expect( + drain(Buffer.alloc(10_000, 0x20), { compressedLimit: 5_000 }), + ).rejects.toMatchObject({ status: 413, reason: "file_size_exceeded" }); + }); +}); diff --git a/src/lib/export/__tests__/restore-backup-round-trip.test.ts b/src/lib/export/__tests__/restore-backup-round-trip.test.ts index 31ff9eed2..a57669b3c 100644 --- a/src/lib/export/__tests__/restore-backup-round-trip.test.ts +++ b/src/lib/export/__tests__/restore-backup-round-trip.test.ts @@ -420,9 +420,15 @@ function recordingTx( return new Proxy({} as Record, { // `$executeRaw` and friends: the restore's per-account lock, and the - // measurement insert, which answer with a row count. + // measurement insert, which answer with a row count. `$queryRaw` is the + // tenant check's read of the schema's foreign keys, which a mock has none + // of (the integration suite runs it against the real catalogue). get: (_t, model: string) => - model.startsWith("$") ? async () => 0 : delegate(model), + model === "$queryRaw" + ? async () => [] + : model.startsWith("$") + ? async () => 0 + : delegate(model), }); } diff --git a/src/lib/export/backup-key-ids.ts b/src/lib/export/backup-key-ids.ts new file mode 100644 index 000000000..44b0ff0e7 --- /dev/null +++ b/src/lib/export/backup-key-ids.ts @@ -0,0 +1,333 @@ +/** + * Which application keys the ciphertext INSIDE a backup was written under. + * + * A disaster-recovery backup carries every encrypted column the way the + * database holds it: a note, a document, a coach message travel as the stored + * ciphertext, base64-encoded, and a restore writes that ciphertext back + * verbatim. The envelope around the whole copy is sealed separately, and key + * rotation re-seals that envelope, but it cannot reach inside it, and it + * cannot reach a copy in the off-host bucket or a file an operator keeps on a + * disk at all. So a backup taken before a rotation still needs the key the + * rotation retired, and a restore without that key writes rows back that no + * reader can open. Nothing in the restore used to notice: the envelope opened, + * the schema passed, and the damage showed up later as notes that would not + * decrypt. + * + * This module answers the question the restore has to ask first: which key + * ids does this file need, and are they all here. Every encrypted column in a + * backup payload is named `…Encrypted`, and each value is one of: + * + * - the string codec as stored in a text column: `.`; + * - a Bytes column, base64-encoded, holding the string codec as UTF-8; + * - a Bytes column, base64-encoded, holding the binary codec + * (`0x02 | keyIdLen | keyId | iv | tag | ct`); + * - a legacy value with no key id, which only the `v1` key opens. + * + * Two readers share the classification: a walk over a parsed document (the + * restore, the preview, the upload and the restore drill already hold one), + * and a text scanner for a backup that is being written, which never exists + * as a document (`createBackupKeyIdTextScanner`). + */ +import { Buffer } from "node:buffer"; + +import { decrypt, decryptBytes, getConfiguredKeyIds } from "@/lib/crypto"; + +/** The key id a value with no key id prefix was written under. */ +export const LEGACY_INNER_KEY_ID = "v1"; + +const KEY_ID = /^[A-Za-z0-9_-]{1,32}$/; +const VERSIONED = /^([A-Za-z0-9_-]{1,32})\.[A-Za-z0-9+/=]+$/; +const BASE64 = /^[A-Za-z0-9+/]+={0,2}$/; +const BASE64_TAIL = /^[A-Za-z0-9+/=]*$/; +const BYTES_CODEC_VERSION = 0x02; +const MIN_CIPHERTEXT_CHARS = 40; + +/** How an inner value is laid out, which decides how a sample is opened. */ +export type InnerCiphertextForm = "string" | "bytes-string" | "binary"; + +export interface InnerCiphertext { + keyId: string; + form: InnerCiphertextForm; +} + +/** + * The key id one inner ciphertext value needs, or null for something that is + * not ciphertext at all (an empty string). Decodes only a prefix, so a + * multi-megabyte document costs the same as a note. + */ +export function innerCiphertextKeyId(value: string): InnerCiphertext | null { + // The shortest ciphertext either codec writes is 28 bytes (nonce and tag), + // 40 characters as base64: anything shorter is not ciphertext, and reading + // it as a legacy value would demand a key for a field that holds none. + if (value.length < MIN_CIPHERTEXT_CHARS) return null; + const versioned = VERSIONED.exec(value.slice(0, 64)); + if (versioned && value.indexOf(".") === versioned[1].length) { + return { keyId: versioned[1], form: "string" }; + } + // A Bytes column, base64-encoded. 48 characters decode to 36 bytes, which + // covers the longest header either codec can have (2 + 32 for the binary + // one, 33 for `.`). + const head = value.slice(0, 48); + if (!BASE64.test(head)) return null; + const bytes = Buffer.from(head, "base64"); + if (bytes.length >= 3 && bytes[0] === BYTES_CODEC_VERSION) { + const keyIdLen = bytes[1]; + if (keyIdLen >= 1 && keyIdLen <= 32 && bytes.length >= 2 + keyIdLen) { + const keyId = bytes.subarray(2, 2 + keyIdLen).toString("latin1"); + if (KEY_ID.test(keyId)) return { keyId, form: "binary" }; + } + } + const text = bytes.toString("latin1"); + const dot = text.indexOf("."); + // The rest of the prefix has to read as base64 too, or a legacy value's + // random bytes could pass for `.` about once in a thousand values. + if ( + dot > 0 && + KEY_ID.test(text.slice(0, dot)) && + BASE64_TAIL.test(text.slice(dot + 1)) + ) { + return { keyId: text.slice(0, dot), form: "bytes-string" }; + } + // No key id anywhere: the legacy layout, which `decrypt` opens with `v1`. + return { keyId: LEGACY_INNER_KEY_ID, form: "bytes-string" }; +} + +/** Whether a member name holds inner ciphertext. */ +export function isEncryptedMember(name: string): boolean { + return name.endsWith("Encrypted") && name.length > "Encrypted".length; +} + +interface KeyUse { + /** How many values need this key. */ + count: number; + /** The top-level sections they sit in (`measurements`, `appSettings`, …). */ + sections: Set; + /** The shortest value seen, kept to prove the key actually opens it. */ + sample: { value: string; form: InnerCiphertextForm } | null; +} + +/** A sample longer than this is not kept; opening it proves nothing more. */ +const MAX_SAMPLE_CHARS = 64 * 1024; + +/** + * Collects the key ids a backup's inner ciphertext needs, with the section + * each one appears in and one sample per key. + */ +export class BackupKeyIdCollector { + private readonly uses = new Map(); + + /** Record one inner value found in `section`. */ + add(section: string, value: string): void { + const found = innerCiphertextKeyId(value); + if (!found) return; + let use = this.uses.get(found.keyId); + if (!use) { + use = { count: 0, sections: new Set(), sample: null }; + this.uses.set(found.keyId, use); + } + use.count += 1; + use.sections.add(section); + if ( + value.length <= MAX_SAMPLE_CHARS && + (!use.sample || value.length < use.sample.value.length) + ) { + use.sample = { value, form: found.form }; + } + } + + /** + * Walk a parsed value and record every `…Encrypted` string in it. `section` + * names where it sits in the file; for a whole document, pass nothing and + * each top-level member is its own section. + */ + visit(value: unknown, section?: string): void { + if (section === undefined) { + if (value && typeof value === "object" && !Array.isArray(value)) { + for (const [key, child] of Object.entries(value)) { + this.walk(child, key, key); + } + } + return; + } + this.walk(value, section, null); + } + + private walk(value: unknown, section: string, member: string | null): void { + if (typeof value === "string") { + if (member !== null && isEncryptedMember(member)) { + this.add(section, value); + } + return; + } + if (Array.isArray(value)) { + // Elements of an array under an `…Encrypted` member are ciphertext too. + for (const element of value) this.walk(element, section, member); + return; + } + if (value && typeof value === "object") { + for (const [key, child] of Object.entries(value)) { + this.walk(child, section, key); + } + } + } + + /** Every key id seen, sorted. */ + keyIds(): string[] { + return [...this.uses.keys()].sort(); + } + + /** How many values need `keyId`. */ + count(keyId: string): number { + return this.uses.get(keyId)?.count ?? 0; + } + + /** @internal for `assessBackupKeys`. */ + entries(): ReadonlyMap { + return this.uses; + } +} + +export interface BackupKeyAssessment { + /** Every key id the file's inner ciphertext was written under. */ + keyIds: string[]; + /** Needed and not in `ENCRYPTION_KEYS` on this host. */ + missing: string[]; + /** + * Configured, but the sample written under it does not open: the id is + * there with different key material behind it. + */ + unreadable: string[]; + /** Values that need a missing or unreadable key. */ + affectedValues: number; +} + +function openSample(sample: { value: string; form: InnerCiphertextForm }) { + switch (sample.form) { + case "string": + decrypt(sample.value); + return; + case "bytes-string": + decrypt(Buffer.from(sample.value, "base64").toString("utf8")); + return; + case "binary": + decryptBytes(Buffer.from(sample.value, "base64")); + return; + } +} + +/** + * Hold what the file needs against what this host has, and open one value per + * key to prove the key under that id is the right one. + * + * `ignoreSections` leaves out sections the caller will not write: the + * instance settings, when the operator has not asked for them back. + */ +export function assessBackupKeys( + collector: BackupKeyIdCollector, + options: { ignoreSections?: ReadonlySet } = {}, +): BackupKeyAssessment { + const ignore = options.ignoreSections ?? new Set(); + const keyIds: string[] = []; + const missing: string[] = []; + const unreadable: string[] = []; + let affectedValues = 0; + // Read lazily: a file with no inner ciphertext needs no key, and must not + // fail on a host (or a test) that has none configured. + let configured: Set | null = null; + for (const [keyId, use] of [...collector.entries()].sort(([a], [b]) => + a.localeCompare(b), + )) { + if ([...use.sections].every((section) => ignore.has(section))) continue; + keyIds.push(keyId); + configured ??= new Set(getConfiguredKeyIds()); + if (!configured.has(keyId)) { + missing.push(keyId); + affectedValues += use.count; + continue; + } + if (!use.sample) continue; + try { + openSample(use.sample); + } catch { + unreadable.push(keyId); + affectedValues += use.count; + } + } + return { keyIds, missing, unreadable, affectedValues }; +} + +/** The refusal code every surface answers a key problem with. */ +export const BACKUP_KEY_MISSING_CODE = "backup.key.missing"; + +/** One sentence an operator can act on, naming the key ids. */ +export function describeBackupKeyProblem( + assessment: BackupKeyAssessment, +): string | null { + const parts: string[] = []; + if (assessment.missing.length > 0) { + const ids = assessment.missing.map((id) => `'${id}'`).join(", "); + parts.push( + `This backup holds ${assessment.affectedValues} encrypted ${ + assessment.affectedValues === 1 ? "value" : "values" + } written under encryption key ${ids}, which is not in ENCRYPTION_KEYS on this server. ` + + `Add ${assessment.missing.length === 1 ? "that key" : "those keys"} back to ENCRYPTION_KEYS and restart, then try again. Nothing was changed.`, + ); + } + if (assessment.unreadable.length > 0) { + const ids = assessment.unreadable.map((id) => `'${id}'`).join(", "); + parts.push( + `Encryption key ${ids} is configured, but it does not open the values in this backup written under that id: ` + + `the key material behind the id differs from the one the backup was written with. Nothing was changed.`, + ); + } + return parts.length > 0 ? parts.join(" ") : null; +} + +/** + * Key ids found in backup JSON as it is written, for a copy that never exists + * as one document. Fed the text in pieces; a member split across two pieces is + * read once the rest of it arrives. + */ +export interface BackupKeyIdTextScanner { + feed(piece: string | Uint8Array): void; + keyIds(): string[]; +} + +const MEMBER_VALUE = + /"([A-Za-z0-9_]*Encrypted)"\s*:\s*"([A-Za-z0-9+/=._-]{0,64})(")?/g; +/** Enough of the previous piece to finish any member that straddles it. */ +const CARRY_CHARS = 256; + +export function createBackupKeyIdTextScanner(): BackupKeyIdTextScanner { + const found = new Set(); + let carry = ""; + return { + feed(piece) { + // latin1, not utf-8: every character this looks for is ASCII, and a + // piece may end in the middle of a multi-byte character. + const text = + carry + + (typeof piece === "string" + ? piece + : Buffer.from( + piece.buffer, + piece.byteOffset, + piece.byteLength, + ).toString("latin1")); + MEMBER_VALUE.lastIndex = 0; + let match: RegExpExecArray | null; + while ((match = MEMBER_VALUE.exec(text)) !== null) { + const [, member, value, closed] = match; + if (!isEncryptedMember(member)) continue; + // A value cut off by the end of the piece is read again next time. + if (!closed && value.length < 64) continue; + const inner = innerCiphertextKeyId(value); + if (inner) found.add(inner.keyId); + } + carry = text.slice(-CARRY_CHARS); + }, + keyIds() { + return [...found].sort(); + }, + }; +} diff --git a/src/lib/export/backup-upload-decode.ts b/src/lib/export/backup-upload-decode.ts new file mode 100644 index 000000000..4db4dc3dd --- /dev/null +++ b/src/lib/export/backup-upload-decode.ts @@ -0,0 +1,116 @@ +/** + * An uploaded backup file's bytes, gunzipped when they are gzip, with both + * ends counted. + * + * The upload counted what arrived and nothing after the gunzip, so a few + * megabytes of gzip could inflate without limit. The JSON scanner keeps every + * section but the measurements in memory, which turned such a file into an + * out-of-memory kill of the whole app. The inflated bytes are now held to + * {@link INFLATION_RATIO} times the compressed bytes read so far (a real + * backup compresses about tenfold), never less than + * {@link MIN_INFLATED_ALLOWANCE}, and never more than {@link MAX_INFLATED_BYTES}. + */ +import { Readable } from "node:stream"; +import { createGunzip } from "node:zlib"; + +/** How far an uploaded gzip may expand, measured on the bytes read so far. */ +export const INFLATION_RATIO = 100; +/** Below this much output the ratio is not what protects the process. */ +export const MIN_INFLATED_ALLOWANCE = 256 * 1024 * 1024; +/** + * The largest file an upload may inflate to. A disaster-recovery backup of + * 1.25 million readings is 662 MB of JSON; this leaves room for several times + * that record and stops a bomb long before memory does. + */ +export const MAX_INFLATED_BYTES = 4 * 1024 * 1024 * 1024; + +/** Thrown for a file that is too large or not valid gzip. */ +export class BackupUploadDecodeError extends Error { + constructor( + readonly status: 413 | 422, + message: string, + readonly reason: + "file_size_exceeded" | "inflated_size_exceeded" | "invalid_gzip", + readonly size?: number, + ) { + super(message); + this.name = "BackupUploadDecodeError"; + } +} + +export interface DecodeLimits { + /** Cap on the bytes as they arrive. */ + compressedLimit: number; + ratio?: number; + minAllowance?: number; + maxInflated?: number; +} + +/** Bytes → bytes, gunzipped when the file starts with the gzip magic. */ +export async function* decodeBackupUpload( + source: AsyncIterable, + limits: DecodeLimits, +): AsyncGenerator { + const ratio = limits.ratio ?? INFLATION_RATIO; + const minAllowance = limits.minAllowance ?? MIN_INFLATED_ALLOWANCE; + const maxInflated = limits.maxInflated ?? MAX_INFLATED_BYTES; + let seen = 0; + async function* counted() { + for await (const chunk of source) { + seen += chunk.byteLength; + if (seen > limits.compressedLimit) { + throw new BackupUploadDecodeError( + 413, + `Upload exceeds ${Math.round(limits.compressedLimit / 1024 / 1024)} MB limit`, + "file_size_exceeded", + seen, + ); + } + yield chunk; + } + } + const iterator = counted()[Symbol.asyncIterator](); + const first = await iterator.next(); + if (first.done) return; + const rest = (async function* () { + yield first.value; + for (;;) { + const next = await iterator.next(); + if (next.done) return; + yield next.value; + } + })(); + const head = first.value; + if (!(head.byteLength >= 2 && head[0] === 0x1f && head[1] === 0x8b)) { + yield* rest; + return; + } + const gunzip = Readable.from(rest).pipe(createGunzip()); + let inflated = 0; + try { + for await (const chunk of gunzip) { + inflated += (chunk as Buffer).byteLength; + const allowed = Math.min( + maxInflated, + Math.max(minAllowance, seen * ratio), + ); + if (inflated > allowed) { + gunzip.destroy(); + throw new BackupUploadDecodeError( + 413, + `The uploaded file expands to more than ${Math.round(allowed / 1024 / 1024)} MB, which no backup this server wrote does. Nothing was stored.`, + "inflated_size_exceeded", + inflated, + ); + } + yield chunk as Buffer; + } + } catch (err) { + if (err instanceof BackupUploadDecodeError) throw err; + throw new BackupUploadDecodeError( + 422, + "Uploaded file is not valid gzip", + "invalid_gzip", + ); + } +} diff --git a/src/lib/export/restore-backup.ts b/src/lib/export/restore-backup.ts index d10100e78..c813fd030 100644 --- a/src/lib/export/restore-backup.ts +++ b/src/lib/export/restore-backup.ts @@ -47,6 +47,17 @@ import { type BackupSource, type StreamedBackup, } from "@/lib/export/streamed-backup"; +import { + assessBackupKeys, + BACKUP_KEY_MISSING_CODE, + describeBackupKeyProblem, +} from "@/lib/export/backup-key-ids"; +import { + assertNoNewForeignReferences, + countForeignReferences, + ForeignReferenceError, + listTenantEdges, +} from "@/lib/export/tenant-boundary"; import { encryptNote } from "@/lib/crypto/note-cipher"; import { encryptToBytes } from "@/lib/ai/coach/bytes-codec"; import { encryptContextToBytes } from "@/lib/labs/biomarker-store"; @@ -240,6 +251,8 @@ export type RestoreFailureCode = | "owner_not_found" | "backup.section.missing" | "document_ciphertext_missing" + | "backup.key.missing" + | "backup.foreign_reference" | "time_budget" | "transaction_failed" | "interrupted" @@ -475,6 +488,40 @@ export async function restoreBackup( ); } + // The ciphertext inside the file is written back verbatim, so every key it + // was written under has to be on this host, and has to be the key that + // wrote it. A copy taken before a key rotation still needs the key the + // rotation retired: the rotation re-seals the stored copy's envelope but + // cannot reach inside it. Refused here, above the first delete, because the + // alternative is an account whose notes, documents and coach history come + // back as rows no reader can open. The instance settings count only when + // the operator asked for them back. + const keyVerdict = assessBackupKeys(streamed.keys, { + ignoreSections: restoreInstanceSettings + ? new Set() + : new Set(["appSettings"]), + }); + const keyProblem = describeBackupKeyProblem(keyVerdict); + if (keyProblem) { + const keyIds = [...keyVerdict.missing, ...keyVerdict.unreadable]; + await auditLog("admin.backups.restore.failed", { + userId: input.actorUserId, + ipAddress: input.ipAddress, + details: { + backupId: backup.id, + ownerId, + reason: "key_missing", + missingKeyIds: keyVerdict.missing, + unreadableKeyIds: keyVerdict.unreadable, + affectedValues: keyVerdict.affectedValues, + }, + }); + return refused(422, BACKUP_KEY_MISSING_CODE, keyProblem, { + errorCode: BACKUP_KEY_MISSING_CODE, + keyIds, + }); + } + // Audit the *intent* before the transaction begins so the trail // describing "an admin is about to restore " survives even // if the operation crashes midway. @@ -605,6 +652,17 @@ export async function restoreBackup( // commit or roll back, then replaces the account again from its own // file, which is what it was asked to do. await tx.$executeRaw`SELECT pg_advisory_xact_lock(hashtext(${`backup-restore:${ownerId}`}))`; + // Every reference this restore writes stays inside the account: the + // references that leave it are counted now and again after the last + // write, and any that grew rolls the whole transaction back + // (`tenant-boundary.ts` says why the check is derived from the schema + // rather than written per section). + const tenantEdges = await listTenantEdges(tx); + const foreignBefore = await countForeignReferences( + tx, + ownerId, + tenantEdges, + ); // Declared INSIDE the transaction so a rollback takes the report with // it. An accumulator that outlived a failed attempt would carry drops // that never happened into the next one. @@ -1835,7 +1893,7 @@ export async function restoreBackup( for (const episode of payload.illnessEpisodes) { if (episode.parentConditionId) { await tx.illnessEpisode.update({ - where: { id: episode.id }, + where: { id: episode.id, userId: ownerId }, data: { parentConditionId: episode.parentConditionId }, }); } @@ -2288,6 +2346,10 @@ export async function restoreBackup( environmentCleared.environmentTravelLocations, ecgRecordings: ecgCleared.ecgRecordings, }; + assertNoNewForeignReferences( + foreignBefore, + await countForeignReferences(tx, ownerId, tenantEdges), + ); progress.sectionsDone = RESTORE_SECTION_STEPS.length; report("sections", true); return { cleared, skipped: summarizeRestoreSkips(skips) }; @@ -2316,6 +2378,12 @@ export async function restoreBackup( }, }); annotate({ meta: { restoreFailReason: verbose } }); + if (err instanceof ForeignReferenceError) { + return refused(422, "backup.foreign_reference", err.message, { + errorCode: "backup.foreign_reference", + references: err.edges, + }); + } if (isStoredBackupReadError(err)) { // The second read, inside the transaction, found the copy replaced or // altered. The transaction rolled back, so nothing was changed. diff --git a/src/lib/export/store-backup-blob.ts b/src/lib/export/store-backup-blob.ts index 36b3b2494..92d5cd785 100644 --- a/src/lib/export/store-backup-blob.ts +++ b/src/lib/export/store-backup-blob.ts @@ -34,6 +34,7 @@ import { type BackupJsonProducer, type PackBackupOptions, } from "@/lib/export/backup-blob"; +import { createBackupKeyIdTextScanner } from "@/lib/export/backup-key-ids"; import { BackupIntegrityError, newChunkStreamId, @@ -112,6 +113,11 @@ export async function storeBackupBlob( await tx.dataBackupChunk.deleteMany({ where: { backupId: row.id } }); const streamId = newChunkStreamId(); + // The key ids the copy's content was written under, read from the JSON + // on its way into the pieces. Rotation re-seals the pieces but never + // what is inside them, so this is what tells an operator how long a + // retired key is still needed (`GET /api/admin/encryption/status`). + const keyScanner = createBackupKeyIdTextScanner(); const { chunks, bytes } = await packBackupChunks( async (sealed, seq) => { await tx.dataBackupChunk.create({ @@ -120,7 +126,11 @@ export async function storeBackupBlob( }); }, streamId, - producer, + (write) => + producer(async (chunk) => { + keyScanner.feed(chunk); + await write(chunk); + }), options, ); @@ -135,6 +145,8 @@ export async function storeBackupBlob( ...(input.ownerAfterRead ? { userId: input.ownerAfterRead() } : {}), chunkCount: chunks, chunkStreamId: streamId, + innerKeyIds: keyScanner.keyIds(), + innerKeyIdsRecorded: true, createdAt: new Date(), }, }); diff --git a/src/lib/export/streamed-backup.ts b/src/lib/export/streamed-backup.ts index e28b9f08d..b87c1b848 100644 --- a/src/lib/export/streamed-backup.ts +++ b/src/lib/export/streamed-backup.ts @@ -21,6 +21,7 @@ * check reads). Run it through `parseBackupPayload` for the rest of the * validation; together that is the whole-file check. */ +import { BackupKeyIdCollector } from "@/lib/export/backup-key-ids"; import { scanBackupJson } from "@/lib/export/backup-json-scan"; import { BACKUP_SCHEMA_VERSION, @@ -51,6 +52,12 @@ export interface StreamedBackup { measurementCount: number; /** The file's size in bytes. */ bytes: number; + /** + * The application key ids the file's inner ciphertext was written under, + * gathered on the first read (`assessBackupKeys` holds them against this + * host's keys). + */ + keys: BackupKeyIdCollector; /** * Read the file again and hand its measurements over in file order, in * batches of at most `size`, each row parsed by the element schema. @@ -93,6 +100,7 @@ export async function readStreamedBackup( ): Promise { let firstWithoutId: number | null = null; let bytes = 0; + const keys = new BackupKeyIdCollector(); async function* counted() { for await (const chunk of source()) { bytes += chunk.byteLength; @@ -104,6 +112,7 @@ export async function readStreamedBackup( onElement: (_key, element, index) => { const row = parseMeasurement(element, index); if (!row.id && firstWithoutId === null) firstWithoutId = index; + keys.visit(element, MEASUREMENTS); options.onMeasurementChecked?.(index + 1); }, }); @@ -120,6 +129,8 @@ export async function readStreamedBackup( ); } + keys.visit(document); + const skipKeys = new Set( Object.keys(document).filter((key) => key !== MEASUREMENTS), ); @@ -128,6 +139,7 @@ export async function readStreamedBackup( raw: document, measurementCount: streamedCounts[MEASUREMENTS] ?? 0, bytes, + keys, async forEachMeasurementBatch(size, fn) { let batch: BackupMeasurement[] = []; await scanBackupJson(source(), { diff --git a/src/lib/export/tenant-boundary.ts b/src/lib/export/tenant-boundary.ts new file mode 100644 index 000000000..6880d8444 --- /dev/null +++ b/src/lib/export/tenant-boundary.ts @@ -0,0 +1,178 @@ +/** + * Every reference a restore writes has to stay inside the account it writes. + * + * A backup file names ids: a custom mood tag names its category, an illness + * names its parent condition, a link names the document and the episode it + * joins. The restore writes those ids back as given, and a foreign key only + * proves the row they point at exists, not whose it is. So a file edited by + * hand, or an upload whose owner is simply what the file says it is, could + * attach the restored account's rows to rows of another account on the same + * host: a custom tag filed under somebody else's category, a note linked to + * somebody else's document. Several of the restore's writes (the tag and + * category upserts keyed on a globally unique `key`, for one) never even look + * at the owner. + * + * Checking each write site by hand is the approach that keeps missing one, so + * the check is derived from the schema instead. Every single-column foreign key + * is read from the database catalogue, and two shapes are held: + * + * - a table with `user_id` pointing at another table with `user_id`: a row of + * the restored account may not point at a row another account owns; + * - a link table without `user_id` joining two such tables: a link whose one + * end is the restored account's may not have its other end in another + * account. + * + * A NULL owner on the far end is a shared catalogue row (a seeded tag) and is + * allowed. The counts are taken twice inside the restore transaction, before + * the first delete and after the last write, and the restore is refused when + * any count went up: the host may have legitimate cross-account links of its + * own (a shared record), and the question is only whether THIS restore added + * one. Refusing throws, which rolls the whole transaction back. + */ +import type { Prisma } from "@/generated/prisma/client"; + +type Db = Pick; + +/** One foreign key, or one pair of them through a link table. */ +export interface TenantEdge { + /** Readable name for the refusal: `child.column -> parent`. */ + name: string; + /** The counting query; `$1` is the owner id. */ + sql: string; +} + +/** + * Identifiers read from the catalogue are spliced into SQL, so each is held + * to this before use (the whitelist-splice rule for `$queryRawUnsafe`). + */ +const IDENT = /^[a-z_][a-z0-9_]*$/; + +interface ForeignKeyRow { + child: string; + child_col: string; + parent: string; + parent_col: string; + child_owned: boolean; + parent_owned: boolean; +} + +/** Thrown inside the restore transaction; rolls it back. */ +export class ForeignReferenceError extends Error { + readonly edges: string[]; + constructor(edges: string[]) { + super( + `The backup references records that belong to another account (${edges.join(", ")}). Nothing was changed.`, + ); + this.name = "ForeignReferenceError"; + this.edges = edges; + } +} + +export async function listTenantEdges(db: Db): Promise { + const rows = await db.$queryRaw` + SELECT + child.relname AS child, + ca.attname AS child_col, + parent.relname AS parent, + pa.attname AS parent_col, + EXISTS ( + SELECT 1 FROM pg_attribute x + WHERE x.attrelid = c.conrelid AND x.attname = 'user_id' + AND NOT x.attisdropped + ) AS child_owned, + EXISTS ( + SELECT 1 FROM pg_attribute x + WHERE x.attrelid = c.confrelid AND x.attname = 'user_id' + AND NOT x.attisdropped + ) AS parent_owned + FROM pg_constraint c + JOIN pg_class child ON child.oid = c.conrelid + JOIN pg_class parent ON parent.oid = c.confrelid + JOIN pg_namespace n ON n.oid = child.relnamespace + JOIN pg_attribute ca ON ca.attrelid = c.conrelid AND ca.attnum = c.conkey[1] + JOIN pg_attribute pa ON pa.attrelid = c.confrelid AND pa.attnum = c.confkey[1] + WHERE c.contype = 'f' + AND array_length(c.conkey, 1) = 1 + AND n.nspname = current_schema() + ORDER BY child.relname, ca.attname + `; + + const safe = rows.filter((row) => + [row.child, row.child_col, row.parent, row.parent_col].every((id) => + IDENT.test(id), + ), + ); + if (safe.length !== rows.length) { + // Never silently narrower than the schema: a check that quietly skips a + // table is the kind that stays green because it matched nothing. + throw new Error( + "Tenant check: a foreign key has an identifier outside [a-z0-9_]", + ); + } + + const edges: TenantEdge[] = []; + for (const fk of safe) { + if (!fk.child_owned || !fk.parent_owned) continue; + edges.push({ + name: `${fk.child}.${fk.child_col} -> ${fk.parent}`, + sql: + `SELECT count(*)::int AS n FROM "${fk.child}" c ` + + `JOIN "${fk.parent}" p ON p."${fk.parent_col}" = c."${fk.child_col}" ` + + `WHERE c.user_id = $1 AND p.user_id IS NOT NULL AND p.user_id <> $1`, + }); + } + + // Link tables: no owner of their own, two or more keys into owned tables. + const byChild = new Map(); + for (const fk of safe) { + if (fk.child_owned || !fk.parent_owned) continue; + const list = byChild.get(fk.child) ?? []; + list.push(fk); + byChild.set(fk.child, list); + } + for (const [child, fks] of byChild) { + if (fks.length < 2) continue; + for (const near of fks) { + for (const far of fks) { + if (near === far) continue; + edges.push({ + name: `${child}.${near.child_col} / ${child}.${far.child_col} -> ${far.parent}`, + sql: + `SELECT count(*)::int AS n FROM "${child}" l ` + + `JOIN "${near.parent}" a ON a."${near.parent_col}" = l."${near.child_col}" ` + + `JOIN "${far.parent}" b ON b."${far.parent_col}" = l."${far.child_col}" ` + + `WHERE a.user_id = $1 AND b.user_id IS NOT NULL AND b.user_id <> $1`, + }); + } + } + } + return edges; +} + +/** How many references from `ownerId`'s rows leave the account, per edge. */ +export async function countForeignReferences( + db: Db, + ownerId: string, + edges: readonly TenantEdge[], +): Promise> { + const counts = new Map(); + for (const edge of edges) { + const [row] = await db.$queryRawUnsafe>( + edge.sql, + ownerId, + ); + counts.set(edge.name, row?.n ?? 0); + } + return counts; +} + +/** Throw when any edge carries more foreign references than it did before. */ +export function assertNoNewForeignReferences( + before: ReadonlyMap, + after: ReadonlyMap, +): void { + const grown = [...after] + .filter(([name, n]) => n > (before.get(name) ?? 0)) + .map(([name]) => name); + if (grown.length > 0) throw new ForeignReferenceError(grown); +} diff --git a/src/lib/import/__tests__/apple-health-staging.test.ts b/src/lib/import/__tests__/apple-health-staging.test.ts new file mode 100644 index 000000000..a8132d17e --- /dev/null +++ b/src/lib/import/__tests__/apple-health-staging.test.ts @@ -0,0 +1,52 @@ +/** + * The sweep removes staged exports no running import can own, and nothing + * else. Mutations that must turn this red: drop the age check (the fresh + * upload of a running import goes), or widen the name pattern (a file the app + * did not write goes). + */ +import { mkdtempSync, readdirSync, utimesSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +import { describe, expect, it } from "vitest"; + +import { + STAGING_MAX_AGE_MS, + sweepStaleImportStaging, +} from "../apple-health-staging"; + +describe("sweepStaleImportStaging", () => { + it("removes old staged uploads and extracted XML, and leaves the rest", async () => { + const dir = mkdtempSync(join(tmpdir(), "healthlog-sweep-")); + const uuid = "0b7f3c2a-1d4e-4f5a-9b8c-7d6e5f4a3b2c"; + const names = { + oldUpload: `healthlog-apple-health-import-${uuid}.bin`, + oldAdminUpload: `healthlog-admin-apple-health-import-${uuid}.bin`, + oldXml: `healthlog-import-${"ab".repeat(12)}.xml`, + freshUpload: `healthlog-upload-${uuid}.bin`, + foreign: "somebody-elses-file.bin", + }; + for (const name of Object.values(names)) { + writeFileSync(join(dir, name), "x"); + } + const now = Date.now(); + const old = new Date(now - STAGING_MAX_AGE_MS - 60_000); + for (const name of [ + names.oldUpload, + names.oldAdminUpload, + names.oldXml, + names.foreign, + ]) { + utimesSync(join(dir, name), old, old); + } + + expect(await sweepStaleImportStaging(dir, now)).toBe(3); + expect(readdirSync(dir).sort()).toEqual( + [names.foreign, names.freshUpload].sort(), + ); + }); + + it("answers zero for a directory that is not there", async () => { + expect(await sweepStaleImportStaging("/nonexistent/healthlog")).toBe(0); + }); +}); diff --git a/src/lib/import/__tests__/unzip-export-xml.test.ts b/src/lib/import/__tests__/unzip-export-xml.test.ts index 10046cbaf..ff9dfc064 100644 --- a/src/lib/import/__tests__/unzip-export-xml.test.ts +++ b/src/lib/import/__tests__/unzip-export-xml.test.ts @@ -6,9 +6,12 @@ import { deflateRawSync } from "node:zlib"; import { pipeline } from "node:stream/promises"; import { Readable } from "node:stream"; +import { readdirSync } from "node:fs"; + import { createByteCap, extractExportXml, + memberOutputCap, readCentralDirectory, } from "../unzip-export-xml"; @@ -22,6 +25,8 @@ function buildMinimalZip( filename: string, payload: Buffer, method: 0 | 8 = 8, + /** What the headers claim the member inflates to; the truth by default. */ + declaredSize?: number, ): Buffer { const compressed = method === 8 ? deflateRawSync(payload) : payload; const crc32 = (() => { @@ -54,7 +59,7 @@ function buildMinimalZip( localHeader.writeUInt16LE(0, 12); // date localHeader.writeUInt32LE(crc32, 14); localHeader.writeUInt32LE(compressed.length, 18); - localHeader.writeUInt32LE(payload.length, 22); + localHeader.writeUInt32LE(declaredSize ?? payload.length, 22); localHeader.writeUInt16LE(nameBuf.length, 26); localHeader.writeUInt16LE(0, 28); // extra len @@ -69,7 +74,7 @@ function buildMinimalZip( cdh.writeUInt16LE(0, 14); // date cdh.writeUInt32LE(crc32, 16); cdh.writeUInt32LE(compressed.length, 20); - cdh.writeUInt32LE(payload.length, 24); + cdh.writeUInt32LE(declaredSize ?? payload.length, 24); cdh.writeUInt16LE(nameBuf.length, 28); cdh.writeUInt16LE(0, 30); // extra cdh.writeUInt16LE(0, 32); // comment @@ -166,6 +171,31 @@ describe("extractExportXml", () => { expect(seenDeclaredBytes).toBe(payload.length); }); + // A member whose headers claim an honest ratio and whose deflate stream + // inflates a thousandfold. The header check passes it; only the count of + // the bytes actually produced, held to the compressed size on disk, stops + // it, and it must leave nothing behind in the staging directory. + it("refuses a member that inflates past its compressed size's cap, whatever it declares", async () => { + const payload = Buffer.alloc(80 * 1024 * 1024); + const compressedLength = deflateRawSync(payload).length; + const zip = buildMinimalZip( + "apple_health_export/export.xml", + payload, + 8, + compressedLength * 2, + ); + const tmp = mkdtempSync(join(tmpdir(), "healthlog-unzip-")); + const zipPath = join(tmp, "export.zip"); + writeFileSync(zipPath, zip); + const staged = () => + readdirSync(tmpdir()).filter((n) => n.startsWith("healthlog-import-")); + const before = new Set(staged()); + + await expect(extractExportXml(zipPath)).rejects.toThrow(/zip bomb/); + expect(staged().filter((n) => !before.has(n))).toEqual([]); + // Inflating past the 64 MiB floor takes a moment on a loaded runner. + }, 60_000); + it("throws when the export.xml member is missing", async () => { const payload = Buffer.from("noop"); const zip = buildMinimalZip("other-file.txt", payload); @@ -209,3 +239,26 @@ describe("createByteCap", () => { ).rejects.toThrow(/exceeds the 6-byte cap/); }); }); + +describe("memberOutputCap", () => { + it("holds a deflated member to 200 times what it occupies, with a floor", () => { + expect( + memberOutputCap({ compressionMethod: 8, compressedSize: 1024 }), + ).toBe(64 * 1024 * 1024); + expect( + memberOutputCap({ + compressionMethod: 8, + compressedSize: 10 * 1024 * 1024, + }), + ).toBe(2000 * 1024 * 1024); + expect( + memberOutputCap({ compressionMethod: 8, compressedSize: 1024 ** 3 }), + ).toBe(8 * 1024 ** 3); + }); + + it("lets a stored member grow by nothing", () => { + expect(memberOutputCap({ compressionMethod: 0, compressedSize: 500 })).toBe( + 500, + ); + }); +}); diff --git a/src/lib/import/apple-health-staging.ts b/src/lib/import/apple-health-staging.ts new file mode 100644 index 000000000..08819a490 --- /dev/null +++ b/src/lib/import/apple-health-staging.ts @@ -0,0 +1,114 @@ +/** + * The staged Apple Health upload, from the request that writes it to the job + * that consumes it, and what happens to it on every way out. + * + * The upload is the person's whole health export in plain text, written to + * the shared temp directory by the kick-off request and read by the import + * worker. Until v1.39.3 several exits left it there: a worker that was not + * running (503), a queue that did not accept the job, an account deleted + * before the worker reached it, and nothing ever swept what was left. Every + * exit now removes it, and a periodic sweep removes anything older than the + * longest an import may run, whatever left it. + * + * It also holds the rule of one running import per account. An import holds + * the archive, the extracted XML (up to 8 GiB) and a worker slot; a second + * one for the same account only doubles that and races the first on the same + * rows. The check and the new row are one transaction under an advisory lock, + * so two uploads that arrive together cannot both pass. + */ +import { readdir, rm, stat } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +import type { Prisma, PrismaClient } from "@/generated/prisma/client"; + +/** An import in one of these holds its staged files and a worker slot. */ +export const ACTIVE_IMPORT_STATUSES = [ + "queued", + "unpacking", + "parsing", + "upserting", +] as const; + +/** + * An active row older than this is not holding anything any more: the queue + * expires an import after six hours (`APPLE_HEALTH_IMPORT_SEND_OPTIONS`) and + * the reconcile pass fails it soon after. It must not block the account. + */ +const ACTIVE_IMPORT_MAX_AGE_MS = 7 * 60 * 60 * 1000; + +/** Staged files older than this belong to no running import. */ +export const STAGING_MAX_AGE_MS = ACTIVE_IMPORT_MAX_AGE_MS; + +/** The names the kick-off and the extractor give their temp files. */ +const STAGING_NAME = + /^healthlog-(?:apple-health-import|admin-apple-health-import|upload)-[0-9a-f-]{36}\.bin$|^healthlog-import-[0-9a-f]{24}\.xml$/; + +/** The error code a second concurrent import is refused with. */ +export const IMPORT_BUSY_CODE = "import.apple_health.busy"; + +export type CreateImportJobResult = + { created: { id: string } } | { busy: { id: string; status: string } }; + +/** + * Create the ImportJob row unless the account already has an import running. + * `data` is the row as the caller would create it. + */ +export async function createImportJobUnlessBusy( + prisma: PrismaClient, + data: Prisma.ImportJobUncheckedCreateInput, + now: Date = new Date(), +): Promise { + return prisma.$transaction(async (tx) => { + await tx.$executeRaw`SELECT pg_advisory_xact_lock(hashtext(${`apple-health-import:${data.userId}`}))`; + const running = await tx.importJob.findFirst({ + where: { + userId: data.userId, + status: { in: [...ACTIVE_IMPORT_STATUSES] }, + startedAt: { gte: new Date(now.getTime() - ACTIVE_IMPORT_MAX_AGE_MS) }, + }, + orderBy: { startedAt: "desc" }, + select: { id: true, status: true }, + }); + if (running) return { busy: running }; + const created = await tx.importJob.create({ data, select: { id: true } }); + return { created }; + }); +} + +/** Remove a staged file. Never throws; a file already gone is fine. */ +export async function discardStagedUpload(path: string): Promise { + await rm(path, { force: true }).catch(() => {}); +} + +/** + * Remove staged uploads and extracted XML files older than + * `STAGING_MAX_AGE_MS` from `dir`. Only names this app writes are touched. + * Answers how many were removed. + */ +export async function sweepStaleImportStaging( + dir: string = tmpdir(), + now: number = Date.now(), +): Promise { + let names: string[]; + try { + names = await readdir(dir); + } catch { + return 0; + } + let removed = 0; + for (const name of names) { + if (!STAGING_NAME.test(name)) continue; + const path = join(dir, name); + try { + const info = await stat(path); + if (!info.isFile()) continue; + if (now - info.mtimeMs < STAGING_MAX_AGE_MS) continue; + await rm(path, { force: true }); + removed++; + } catch { + // Gone between the listing and here, or not ours to remove. + } + } + return removed; +} diff --git a/src/lib/import/unzip-export-xml.ts b/src/lib/import/unzip-export-xml.ts index e9922a7ad..4f046ef57 100644 --- a/src/lib/import/unzip-export-xml.ts +++ b/src/lib/import/unzip-export-xml.ts @@ -73,6 +73,39 @@ const MAX_DECOMPRESSED_BYTES = 8 * 1024 * 1024 * 1024; */ const MAX_COMPRESSION_RATIO = 200; +/** + * Floor for the output cap of a small member. The ratio cap alone would + * refuse a tiny, legitimately very compressible export; below this many + * bytes of output the ratio is not what protects the disk. + */ +const MIN_OUTPUT_CAP_BYTES = 64 * 1024 * 1024; + +/** + * How many bytes one member may inflate to, from what it ACTUALLY occupies + * in the archive rather than what its header claims. The header is the + * attacker's to write: a member declaring an honest-looking 10:1 ratio can + * still inflate at 1000:1, and with only the flat 8 GiB ceiling an 8 MB + * upload could fill `/tmp` with 8 GB before anything noticed. The streamed + * byte count is held to the ratio the preflight states, applied to the + * compressed bytes on disk, and never above the absolute ceiling. A stored + * member cannot grow at all. + */ +export function memberOutputCap(entry: { + compressionMethod: number; + compressedSize: number; +}): number { + if (entry.compressionMethod === 0) { + return Math.min(entry.compressedSize, MAX_DECOMPRESSED_BYTES); + } + return Math.min( + MAX_DECOMPRESSED_BYTES, + Math.max( + MIN_OUTPUT_CAP_BYTES, + entry.compressedSize * MAX_COMPRESSION_RATIO, + ), + ); +} + /** * Cap on the central-directory slice the extractor is willing to hold * in memory. A real Apple export's central directory is a few KB (one @@ -636,7 +669,7 @@ async function streamEntryToFile( } const dest = createWriteStream(destPath); - const cap = createByteCap(MAX_DECOMPRESSED_BYTES); + const cap = createByteCap(memberOutputCap(entry)); try { if (entry.compressedSize === 0) { diff --git a/src/lib/jobs/__tests__/apple-health-import-reconcile.test.ts b/src/lib/jobs/__tests__/apple-health-import-reconcile.test.ts index 219978aae..cff18e33f 100644 --- a/src/lib/jobs/__tests__/apple-health-import-reconcile.test.ts +++ b/src/lib/jobs/__tests__/apple-health-import-reconcile.test.ts @@ -24,6 +24,10 @@ const mocks = vi.hoisted(() => ({ getJobById: vi.fn(), })); +vi.mock("@/lib/import/apple-health-staging", () => ({ + sweepStaleImportStaging: vi.fn(async () => 2), +})); + vi.mock("@/lib/db", () => ({ prisma: { importJob: { @@ -172,7 +176,7 @@ describe("handleImportJobReconcileTick", () => { await expect(handleImportJobReconcileTick([] as never)).resolves.toEqual({ ok: true, - did: {}, + did: { import_staging_swept: 2 }, }); expect(mocks.findMany).toHaveBeenCalledTimes(1); }); diff --git a/src/lib/jobs/__tests__/backup-restore-outcome.test.ts b/src/lib/jobs/__tests__/backup-restore-outcome.test.ts index 121a8d614..b1c8f82d7 100644 --- a/src/lib/jobs/__tests__/backup-restore-outcome.test.ts +++ b/src/lib/jobs/__tests__/backup-restore-outcome.test.ts @@ -37,6 +37,8 @@ const CODE_TABLE: Record = { owner_not_found: true, "backup.section.missing": true, document_ciphertext_missing: true, + "backup.key.missing": true, + "backup.foreign_reference": true, time_budget: true, transaction_failed: true, interrupted: true, diff --git a/src/lib/jobs/__tests__/offhost-backup.test.ts b/src/lib/jobs/__tests__/offhost-backup.test.ts index a9d07113c..d038b3e73 100644 --- a/src/lib/jobs/__tests__/offhost-backup.test.ts +++ b/src/lib/jobs/__tests__/offhost-backup.test.ts @@ -32,6 +32,14 @@ import { const ENC_KEY = "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"; +/** The purge ledger and key-use rows the run touches after each upload. */ +function purgeLedgerMocks() { + return { + offhostPurgeRequest: { count: vi.fn().mockResolvedValue(0) }, + offhostBackupKeyUse: { upsert: vi.fn() }, + }; +} + describe("offhost-backup envelope", () => { it("encrypts and decrypts JSON round-trip", () => { const key = Buffer.from(ENC_KEY, "hex"); @@ -184,8 +192,10 @@ describe("runOffhostBackup", () => { s3.store.set("2020-01-01/user-old.json.enc", Buffer.from([0])); const prisma = { + ...purgeLedgerMocks(), offhostBackupState: { upsert: vi.fn() }, user: { + count: vi.fn().mockResolvedValue(1), findMany: vi.fn().mockResolvedValue([{ id: "u1" }, { id: "u2" }]), }, measurement: { findMany: vi.fn().mockResolvedValue([]) }, @@ -214,7 +224,11 @@ describe("runOffhostBackup", () => { expect(s3.deleteObject).not.toHaveBeenCalled(); const ct = s3.store.get("2026-05-08/user-u1.json.enc")!; - const decoded = decryptBackup(ct, Buffer.from(ENC_KEY, "hex")); + const decoded = decryptBackup( + ct, + Buffer.from(ENC_KEY, "hex"), + "2026-05-08/user-u1.json.enc", + ); const parsed = JSON.parse(decoded); expect(parsed.userId).toBe("u1"); expect(() => backupPayloadSchema.parse(parsed)).not.toThrow(); @@ -248,11 +262,43 @@ describe("runOffhostBackup", () => { expect(first.create.lastSuccessAt).toBeInstanceOf(Date); }); + it("takes back the copy of an account deleted while it was written", async () => { + const s3 = makeS3Mock(); + const ledger = purgeLedgerMocks(); + const prisma = { + ...ledger, + offhostBackupState: { upsert: vi.fn() }, + user: { + // u1 is gone by the time its upload lands; u2 is not. + count: vi.fn(async ({ where }: { where: { id: string } }) => + where.id === "u1" ? 0 : 1, + ), + findMany: vi.fn().mockResolvedValue([{ id: "u1" }, { id: "u2" }]), + }, + }; + const report = await runOffhostBackup( + // eslint-disable-next-line @typescript-eslint/no-explicit-any + prisma as any, + s3, + new Date("2026-05-08T03:00:00Z"), + ); + expect(s3.store.has("2026-05-08/user-u1.json.enc")).toBe(false); + expect(s3.store.has("2026-05-08/user-u2.json.enc")).toBe(true); + expect(s3.deleteObject).toHaveBeenCalledWith("2026-05-08/user-u1.json.enc"); + expect(report.uploaded).toBe(1); + // No ledger row for an account that no longer exists. + expect(prisma.offhostBackupState.upsert).toHaveBeenCalledTimes(1); + }); + it("uploads the canonical builder output without reshaping it", async () => { const s3 = makeS3Mock(); const prisma = { + ...purgeLedgerMocks(), offhostBackupState: { upsert: vi.fn() }, - user: { findMany: vi.fn().mockResolvedValue([{ id: "u1" }]) }, + user: { + count: vi.fn().mockResolvedValue(1), + findMany: vi.fn().mockResolvedValue([{ id: "u1" }]), + }, }; const canonicalPayload = { schemaVersion: BACKUP_SCHEMA_VERSION, @@ -301,7 +347,13 @@ describe("runOffhostBackup", () => { expect(report.uploaded).toBe(1); const ciphertext = s3.store.get("2026-05-08/user-u1.json.enc")!; expect( - JSON.parse(decryptBackup(ciphertext, Buffer.from(ENC_KEY, "hex"))), + JSON.parse( + decryptBackup( + ciphertext, + Buffer.from(ENC_KEY, "hex"), + "2026-05-08/user-u1.json.enc", + ), + ), ).toEqual(canonicalPayload); expect(() => backupPayloadSchema.parse(canonicalPayload)).not.toThrow(); }); @@ -310,8 +362,10 @@ describe("runOffhostBackup", () => { const s3 = makeS3Mock(); const upsert = vi.fn(); const prisma = { + ...purgeLedgerMocks(), offhostBackupState: { upsert }, user: { + count: vi.fn().mockResolvedValue(1), findMany: vi.fn().mockResolvedValue([{ id: "u1" }, { id: "u2" }]), }, }; @@ -357,10 +411,14 @@ describe("runOffhostBackup", () => { it("still reports a landed object as uploaded when the ledger write fails", async () => { const s3 = makeS3Mock(); const prisma = { + ...purgeLedgerMocks(), offhostBackupState: { upsert: vi.fn().mockRejectedValue(new Error("pool timeout")), }, - user: { findMany: vi.fn().mockResolvedValue([{ id: "u1" }]) }, + user: { + count: vi.fn().mockResolvedValue(1), + findMany: vi.fn().mockResolvedValue([{ id: "u1" }]), + }, }; const report = await runOffhostBackup( @@ -382,8 +440,12 @@ describe("runOffhostBackup", () => { it("refuses an account whose object outgrows one multipart upload", async () => { const s3 = makeS3Mock(); const prisma = { + ...purgeLedgerMocks(), offhostBackupState: { upsert: vi.fn() }, - user: { findMany: vi.fn().mockResolvedValue([{ id: "u1" }]) }, + user: { + count: vi.fn().mockResolvedValue(1), + findMany: vi.fn().mockResolvedValue([{ id: "u1" }]), + }, }; mocks.buildFullBackupPayload.mockResolvedValue({ payload: { @@ -419,8 +481,12 @@ describe("runOffhostBackup", () => { it("reports the largest object it wrote", async () => { const s3 = makeS3Mock(); const prisma = { + ...purgeLedgerMocks(), offhostBackupState: { upsert: vi.fn() }, - user: { findMany: vi.fn().mockResolvedValue([{ id: "u1" }]) }, + user: { + count: vi.fn().mockResolvedValue(1), + findMany: vi.fn().mockResolvedValue([{ id: "u1" }]), + }, }; const report = await runOffhostBackup( prisma as never, @@ -453,8 +519,8 @@ describe("uploadEncryptedBackup", () => { const stored = s3.store.get("k")!; expect(stored.byteLength).toBe(bytes); - expect(stored.subarray(0, 5).toString("binary")).toBe("HLBK\x03"); - expect(decryptBackup(stored, key)).toBe(document); + expect(stored.subarray(0, 5).toString("binary")).toBe("HLBK\x04"); + expect(decryptBackup(stored, key, "k")).toBe(document); }); it("restores an object of the old shape and one of the new one alike", async () => { @@ -470,10 +536,53 @@ describe("uploadEncryptedBackup", () => { const oldBytes = s3.store.get("old")!; const newBytes = s3.store.get("new")!; expect(oldBytes.subarray(0, 5).toString("binary")).toBe("HLBK\x02"); - expect(newBytes.subarray(0, 5).toString("binary")).toBe("HLBK\x03"); + expect(newBytes.subarray(0, 5).toString("binary")).toBe("HLBK\x04"); // Different framing, same record, one reader. expect(decryptBackup(oldBytes, key)).toBe(document); - expect(decryptBackup(newBytes, key)).toBe(document); + expect(decryptBackup(newBytes, key, "new")).toBe(document); + }); + + it("binds a new object to its key in the bucket", async () => { + const s3 = makeS3Mock(); + const document = JSON.stringify({ userId: "u1" }); + await uploadEncryptedBackup( + s3, + "2026-09-01/user-u1.json.enc", + key, + (write) => write(document), + ); + const stored = s3.store.get("2026-09-01/user-u1.json.enc")!; + expect(decryptBackup(stored, key, "2026-09-01/user-u1.json.enc")).toBe( + document, + ); + // The same bytes presented as another account's copy, or another night's, + // do not open. + expect(() => + decryptBackup(stored, key, "2026-09-01/user-u2.json.enc"), + ).toThrow(); + expect(() => + decryptBackup(stored, key, "2026-09-02/user-u1.json.enc"), + ).toThrow(); + }); + + it("reads objects under a retired key through the previous-keys ring", async () => { + const s3 = makeS3Mock(); + const document = JSON.stringify({ userId: "u1" }); + const oldKey = Buffer.alloc(32, 7); + const newKey = Buffer.alloc(32, 9); + await uploadEncryptedBackup(s3, "k4", oldKey, (write) => write(document)); + await s3.putObject("k2", encryptBackup(document, oldKey)); + const ring = { active: newKey, previous: [oldKey] }; + expect(decryptBackup(s3.store.get("k4")!, ring, "k4")).toBe(document); + expect(decryptBackup(s3.store.get("k2")!, ring)).toBe(document); + // Without the retired key the new-format object names the key it needs. + expect(() => + decryptBackup( + s3.store.get("k4")!, + { active: newKey, previous: [] }, + "k4", + ), + ).toThrow(/BACKUP_ENCRYPTION_PREVIOUS_KEYS/); }); it("rejects a tampered version-3 object rather than returning a partial one", async () => { diff --git a/src/lib/jobs/__tests__/offhost-purge.test.ts b/src/lib/jobs/__tests__/offhost-purge.test.ts new file mode 100644 index 000000000..eb6224df9 --- /dev/null +++ b/src/lib/jobs/__tests__/offhost-purge.test.ts @@ -0,0 +1,189 @@ +/** + * Off-host copies leave the bucket with the account. + * + * The bucket is a map behind the same `S3Like` boundary the real client + * implements; the request ledger is an in-memory table. Mutations that must + * turn this red: drop the `deleteObject` call (the objects stay), remove the + * request on failure too (the refusal case loses the request), or match on the + * key prefix instead of the full account pattern (another account's objects, + * or the health-check probe, go too). + */ +import { beforeEach, describe, expect, it, vi } from "vitest"; + +vi.mock("@/lib/auth/audit", () => ({ auditLog: vi.fn() })); + +import { auditLog } from "@/lib/auth/audit"; +import { processOffhostPurges, requestOffhostPurge } from "../offhost-purge"; + +interface Row { + id: string; + subjectId: string; + reason: string; + requestedAt: Date; + attempts: number; + lastAttemptAt: Date | null; + lastFailure: string | null; +} + +function makeLedger(existingUsers: string[] = []) { + const rows: Row[] = []; + let seq = 0; + const prisma = { + offhostPurgeRequest: { + create: vi.fn( + async ({ data }: { data: { subjectId: string; reason: string } }) => { + const row: Row = { + id: `r${++seq}`, + subjectId: data.subjectId, + reason: data.reason, + requestedAt: new Date(Date.now() + seq), + attempts: 0, + lastAttemptAt: null, + lastFailure: null, + }; + rows.push(row); + return { id: row.id }; + }, + ), + findMany: vi.fn(async () => + [...rows].sort((a, b) => +a.requestedAt - +b.requestedAt), + ), + delete: vi.fn(async ({ where }: { where: { id: string } }) => { + rows.splice( + rows.findIndex((r) => r.id === where.id), + 1, + ); + }), + updateMany: vi.fn( + async ({ + where, + data, + }: { + where: { id: { in: string[] } }; + data: { lastAttemptAt: Date; lastFailure: string }; + }) => { + for (const row of rows) { + if (!where.id.in.includes(row.id)) continue; + row.attempts += 1; + row.lastAttemptAt = data.lastAttemptAt; + row.lastFailure = data.lastFailure; + } + }, + ), + }, + user: { + count: vi.fn(async ({ where }: { where: { id: string } }) => + existingUsers.includes(where.id) ? 1 : 0, + ), + }, + }; + return { rows, prisma }; +} + +function makeBucket(keys: string[]) { + const store = new Set(keys); + return { + store, + putObject: vi.fn(), + putStream: vi.fn(), + getObject: vi.fn(), + headObject: vi.fn(), + listObjects: vi.fn(async (prefix: string) => + [...store].filter((k) => k.startsWith(prefix)).map((key) => ({ key })), + ), + deleteObject: vi.fn(async (key: string) => { + store.delete(key); + }), + }; +} + +const BUCKET = [ + "2026-09-01/user-gone.json.enc", + "2026-09-02/user-gone.json.enc", + "2026-09-02/user-gone2.json.enc", + "2026-09-02/user-stays.json.enc", + "_healthcheck/1.bin", +]; + +beforeEach(() => { + vi.unstubAllEnvs(); + vi.clearAllMocks(); + vi.stubEnv("BACKUP_S3_ENDPOINT", "https://r2.example"); + vi.stubEnv("BACKUP_S3_BUCKET", "hl-backups"); + vi.stubEnv("BACKUP_S3_ACCESS_KEY", "AKIA"); + vi.stubEnv("BACKUP_S3_SECRET_KEY", "secret"); + vi.stubEnv("BACKUP_ENCRYPTION_KEY", "ab".repeat(32)); +}); + +describe("off-host purge", () => { + it("deletes every copy of the account, and nothing else", async () => { + const { rows, prisma } = makeLedger(); + await requestOffhostPurge(prisma as never, "gone", "account_deleted"); + const bucket = makeBucket(BUCKET); + + const report = await processOffhostPurges(prisma as never, bucket); + + expect([...bucket.store].sort()).toEqual([ + "2026-09-02/user-gone2.json.enc", + "2026-09-02/user-stays.json.enc", + "_healthcheck/1.bin", + ]); + expect(report).toMatchObject({ + completed: 1, + failed: 0, + objectsDeleted: 2, + }); + expect(rows).toHaveLength(0); + // The account is gone, and so is its audit trail: the receipt carries the + // reason and the count, not the account. + expect(auditLog).toHaveBeenCalledWith( + "offhost.backup.purged", + expect.objectContaining({ + userId: null, + details: expect.objectContaining({ + reason: "account_deleted", + objectsDeleted: 2, + }), + }), + ); + }); + + it("keeps the request, with the bucket's answer, when a delete is refused", async () => { + const { rows, prisma } = makeLedger(["gone"]); + await requestOffhostPurge(prisma as never, "gone", "data_wiped"); + const bucket = makeBucket(BUCKET); + bucket.deleteObject.mockRejectedValueOnce(new Error("AccessDenied")); + + const report = await processOffhostPurges(prisma as never, bucket); + + expect(report).toMatchObject({ completed: 0, failed: 1 }); + expect(rows).toHaveLength(1); + expect(rows[0]).toMatchObject({ attempts: 1, lastFailure: "AccessDenied" }); + + // The next run finishes it, and the account that still exists keeps the + // receipt. + const again = await processOffhostPurges(prisma as never, bucket); + expect(again.completed).toBe(1); + expect(rows).toHaveLength(0); + expect(auditLog).toHaveBeenCalledWith( + "offhost.backup.purged", + expect.objectContaining({ userId: "gone" }), + ); + }); + + it("records nothing on a host without off-host backup", async () => { + vi.stubEnv("BACKUP_S3_ENDPOINT", ""); + const { rows, prisma } = makeLedger(); + expect( + await requestOffhostPurge(prisma as never, "gone", "account_deleted"), + ).toBe(false); + expect(rows).toHaveLength(0); + }); + + it("does not list the bucket when nothing is pending", async () => { + const { prisma } = makeLedger(); + const bucket = makeBucket(BUCKET); + await processOffhostPurges(prisma as never, bucket); + expect(bucket.listObjects).not.toHaveBeenCalled(); + }); +}); diff --git a/src/lib/jobs/__tests__/restore-drill.test.ts b/src/lib/jobs/__tests__/restore-drill.test.ts index e5367f857..41e1cc676 100644 --- a/src/lib/jobs/__tests__/restore-drill.test.ts +++ b/src/lib/jobs/__tests__/restore-drill.test.ts @@ -1,5 +1,6 @@ import type { Readable } from "node:stream"; import { describe, it, expect, beforeEach, vi } from "vitest"; +import { _resetCryptoCacheForTests, encrypt } from "@/lib/crypto"; import { encryptBackup } from "../offhost-backup"; import { handleRestoreDrill, @@ -115,6 +116,43 @@ describe("runRestoreDrill", () => { expect(report.stale).toBe(true); }); + it("decrypts a value per inner key id and names a key the host lacks", async () => { + vi.stubEnv("ENCRYPTION_KEY", ""); + vi.stubEnv( + "ENCRYPTION_KEYS", + JSON.stringify({ old: "11".repeat(32), cur: "22".repeat(32) }), + ); + vi.stubEnv("ENCRYPTION_ACTIVE_KEY_ID", "old"); + _resetCryptoCacheForTests(); + const note = Buffer.from(encrypt("a note"), "utf8").toString("base64"); + const s3 = makeS3Mock({ + "2026-06-01/user-abc.json.enc": backupObject({ + measurements: [{ id: "m1", notesEncrypted: note }], + }), + }); + const now = new Date("2026-06-02T04:11:00Z"); + const report = await runRestoreDrill(s3, now); + expect(report.innerKeyIds).toEqual(["old"]); + + // The rotation ran, and the operator dropped the old key. + vi.stubEnv("ENCRYPTION_KEYS", JSON.stringify({ cur: "22".repeat(32) })); + vi.stubEnv("ENCRYPTION_ACTIVE_KEY_ID", "cur"); + _resetCryptoCacheForTests(); + await expect(runRestoreDrill(s3, now)).rejects.toThrow(/'old'/); + + // Same id, different key material behind it. + vi.stubEnv( + "ENCRYPTION_KEYS", + JSON.stringify({ old: "33".repeat(32), cur: "22".repeat(32) }), + ); + _resetCryptoCacheForTests(); + await expect(runRestoreDrill(s3, now)).rejects.toThrow( + /does not open the values/, + ); + vi.unstubAllEnvs(); + _resetCryptoCacheForTests(); + }); + it("throws when the bucket holds no backup-shaped objects", async () => { const s3 = makeS3Mock({ "_healthcheck/1.bin": Buffer.from([0x42]) }); await expect(runRestoreDrill(s3)).rejects.toThrow(/no backup objects/); diff --git a/src/lib/jobs/apple-health-import-worker.ts b/src/lib/jobs/apple-health-import-worker.ts index 4105262c0..ee4d6af78 100644 --- a/src/lib/jobs/apple-health-import-worker.ts +++ b/src/lib/jobs/apple-health-import-worker.ts @@ -49,6 +49,7 @@ import { } from "@/lib/rollups/measurement-rollups"; import { withBackgroundEvent } from "@/lib/logging/background"; import { DEFAULT_TIMEZONE } from "@/lib/tz/format"; +import { sweepStaleImportStaging } from "@/lib/import/apple-health-staging"; /** * Queue + cron for the periodic orphan-ImportJob sweep. v1.32.1 @@ -90,13 +91,18 @@ export async function handleImportJobReconcileTick( return withBackgroundEvent( "job.apple_health_import_reconcile", async (evt) => { + // Staged uploads and extracted XML that no running import can still + // own: whatever exit left them (a crash, a restart mid-import, a + // deploy), the person's export does not stay in /tmp past this. + const swept = await sweepStaleImportStaging().catch(() => 0); + evt.addMeta("import_staging_swept", swept); try { await reconcileOrphanImportJobs(); } catch (err) { evt.addWarning(`apple-health-import-reconcile failed: ${err}`); return jobFailed("apple health import reconcile failed", err); } - return jobDone(); + return jobDone({ import_staging_swept: swept }); }, ); } @@ -327,16 +333,31 @@ export async function handleAppleHealthImport( `[apple-health-import] No ImportJob row for pgBossJobId=${job.id};` + " creating a stand-in", ); - importJob = await prisma.importJob.create({ - data: { - userId, - triggeredByAdminId: triggeredByAdminId ?? null, - pgBossJobId: job.id, - status: "queued", - uploadBytes, - parserRevision: APPLE_HEALTH_IMPORT_PARSER_REVISION, - }, - }); + // The row is gone when the account is: an account deleted between the + // upload and this run takes its ImportJob with it by cascade. Then there + // is nobody to import for, and the staged export is removed rather than + // left in /tmp by a stand-in create that fails on the missing account. + const accountExists = + (await prisma.user.count({ where: { id: userId } })) > 0; + if (!accountExists) { + safeUnlink(uploadPath); + return jobDone({ skipped: "account_gone" }); + } + try { + importJob = await prisma.importJob.create({ + data: { + userId, + triggeredByAdminId: triggeredByAdminId ?? null, + pgBossJobId: job.id, + status: "queued", + uploadBytes, + parserRevision: APPLE_HEALTH_IMPORT_PARSER_REVISION, + }, + }); + } catch (err) { + safeUnlink(uploadPath); + throw err; + } } const importJobId = importJob.id; @@ -588,16 +609,26 @@ export async function handleAppleHealthImport( : err instanceof Error ? err.message : String(err); - await prisma.importJob.update({ - where: { id: importJobId }, - data: { - status: "failed", - failureReason: reason.slice(0, 1000), - completedAt: new Date(), - }, - }); + // The files first: the row update below throws when the account was + // deleted mid-import (the row went with it), and the export must not be + // left behind in that case either. if (extractedXmlPath) safeUnlink(extractedXmlPath); safeUnlink(uploadPath); + try { + await prisma.importJob.update({ + where: { id: importJobId }, + data: { + status: "failed", + failureReason: reason.slice(0, 1000), + completedAt: new Date(), + }, + }); + } catch (updateErr) { + console.warn( + `[apple-health-import] Could not record the failure on ImportJob=${importJobId}`, + updateErr, + ); + } throw err; } } diff --git a/src/lib/jobs/backup-restore.ts b/src/lib/jobs/backup-restore.ts index e0b3595eb..16519d664 100644 --- a/src/lib/jobs/backup-restore.ts +++ b/src/lib/jobs/backup-restore.ts @@ -120,12 +120,15 @@ export interface BackupRestoreFailure { message: string; /** The sections a file's manifest names and it does not carry. */ sections?: string[]; + /** The encryption key ids a file needs and this host cannot open it with. */ + keyIds?: string[]; } const failureSchema = z.object({ code: z.string(), message: z.string(), sections: z.array(z.string()).optional(), + keyIds: z.array(z.string()).optional(), }); const progressSchema = z.object({ @@ -308,6 +311,7 @@ function failureJson(failure: BackupRestoreFailure) { code: failure.code, message: failure.message, ...(failure.sections ? { sections: failure.sections } : {}), + ...(failure.keyIds ? { keyIds: failure.keyIds } : {}), }); } @@ -632,11 +636,15 @@ export async function runBackupRestoreJob( const sections = Array.isArray(outcome.meta?.sections) ? (outcome.meta.sections as unknown[]).map(String) : undefined; + const keyIds = Array.isArray(outcome.meta?.keyIds) + ? (outcome.meta.keyIds as unknown[]).map(String) + : undefined; await finish("failed", { failure: { code: outcome.code, message: outcome.message, ...(sections ? { sections } : {}), + ...(keyIds ? { keyIds } : {}), }, }); // A file refused by its checks is the operator's to fix and fails the diff --git a/src/lib/jobs/job-outcome.ts b/src/lib/jobs/job-outcome.ts index 3df4416ce..eadaa01e7 100644 --- a/src/lib/jobs/job-outcome.ts +++ b/src/lib/jobs/job-outcome.ts @@ -107,6 +107,7 @@ export const JOB_FACT_ALLOWLIST: ReadonlySet = new Set([ "geolite2_fetch_status", "host_metric_pruned", "hourly_rows_upserted", + "import_staging_swept", "imported", "in_slot", "in_window", @@ -144,6 +145,10 @@ export const JOB_FACT_ALLOWLIST: ReadonlySet = new Set([ "offhost_backup_oversized", "offhost_backup_total_users", "offhost_backup_uploaded", + "offhost_purge_completed", + "offhost_purge_failed", + "offhost_purge_objects_deleted", + "offhost_purge_pending", "outcome", "passes_failed", "per_sample_rows_deleted", diff --git a/src/lib/jobs/offhost-backup.ts b/src/lib/jobs/offhost-backup.ts index cf119d656..5317d450f 100644 --- a/src/lib/jobs/offhost-backup.ts +++ b/src/lib/jobs/offhost-backup.ts @@ -11,23 +11,29 @@ * Object key layout: * //user-.json.enc * - * Retention: the worker NEVER calls DeleteObject on backup keys. Operators - * MUST configure a bucket-level lifecycle rule (e.g. expire after - * `BACKUP_RETENTION_DAYS`, which this module reads nowhere — see - * `loadOffhostConfig`). This keeps the IAM grant for the worker - * limited to PutObject + GetObject + AbortMultipartUpload, so a compromised - * worker cannot wipe the backup history. The abort is what cleans up a run - * that failed partway rather than leaving billed, unlistable parts behind; - * `AbortMultipartUpload` can only touch an upload this worker started, never - * a finished object. See docs/ops/backup-restore.md. + * Retention: the worker does not expire objects by age. Operators configure a + * bucket-level lifecycle rule (e.g. expire after `BACKUP_RETENTION_DAYS`, + * which this module reads nowhere — see `loadOffhostConfig`); the admin + * off-host card reads that rule back (`probeOffhostLifecycle`) and says when + * there is none. Objects ARE deleted in one case: an account that is deleted + * or whose data is wiped (`offhost-purge.ts`), which needs DeleteObject in the + * grant. `AbortMultipartUpload` cleans up a run that failed partway rather + * than leaving billed, unlistable parts behind. See + * docs/ops/backup-restore.md. */ import { envOr } from "@/lib/env"; import { Buffer } from "node:buffer"; -import { createCipheriv, createDecipheriv, randomBytes } from "node:crypto"; +import { + createCipheriv, + createDecipheriv, + createHash, + randomBytes, +} from "node:crypto"; import { Readable, Transform } from "node:stream"; import { createGunzip, createGzip, gunzipSync, gzipSync } from "node:zlib"; import type { PrismaClient } from "@/generated/prisma/client"; import { createRawStreamEncryptor, decryptRawStream } from "@/lib/crypto"; +import { createBackupKeyIdTextScanner } from "@/lib/export/backup-key-ids"; import { streamFullBackupJson } from "@/lib/export/full-backup-stream"; import { annotate, getEvent } from "@/lib/logging/context"; @@ -41,7 +47,16 @@ export interface OffhostBackupConfig { accessKey: string; secretKey: string; region: string; + /** The key new objects are written under (`BACKUP_ENCRYPTION_KEY`). */ encryptionKey: Buffer; + /** + * Keys older objects may still be under (`BACKUP_ENCRYPTION_PREVIOUS_KEYS`, + * comma-separated). Read only; nothing is written under them. This is the + * rotation path: the new key goes into `BACKUP_ENCRYPTION_KEY`, the old one + * here, until the bucket's lifecycle rule has retired every object written + * under it. + */ + previousEncryptionKeys: Buffer[]; } export class OffhostBackupNotConfiguredError extends Error { @@ -94,8 +109,8 @@ export function loadOffhostConfig(): OffhostBackupConfig | null { // `BACKUP_RETENTION_DAYS` is deliberately absent from this config. It used // to be parsed and clamped here and then read by nobody, which read as an // enforcer the worker is not: retention belongs to the bucket's lifecycle - // rule, and no consumer can ever appear here because DeleteObject is kept - // out of the worker's grant on purpose (see the header). The variable stays + // rule, which the worker leaves to the bucket rather than deleting by age + // (see the header). The variable stays // documented and on the compose whitelist because it is the number the // operator sets that rule to. return { @@ -105,6 +120,11 @@ export function loadOffhostConfig(): OffhostBackupConfig | null { secretKey, region: envOr("BACKUP_S3_REGION", "auto"), encryptionKey: decodeBackupKey(encRaw), + previousEncryptionKeys: (process.env.BACKUP_ENCRYPTION_PREVIOUS_KEYS ?? "") + .split(",") + .map((raw) => raw.trim()) + .filter((raw) => raw.length > 0) + .map(decodeBackupKey), }; } @@ -115,6 +135,9 @@ export function loadOffhostConfig(): OffhostBackupConfig | null { * 1: magic(4)="HLBK" || 0x01 || iv(12) || tag(16) || ciphertext(json) * 2: magic(4)="HLBK" || 0x02 || iv(12) || tag(16) || ciphertext(gzip(json)) * 3: magic(4)="HLBK" || 0x03 || iv(12) || ciphertext(gzip(json)) || tag(16) + * 4: magic(4)="HLBK" || 0x04 || keyIdLen(1) || keyId || iv(12) + * || ciphertext(gzip(json)) || tag(16), with the object's own key + * (`/user-.json.enc`) as GCM associated data * * Version 1 encrypted the JSON directly; version 2 gzipped it first. Both put * the tag in front of the ciphertext, and that is precisely what could not be @@ -135,10 +158,108 @@ export function loadOffhostConfig(): OffhostBackupConfig | null { const BACKUP_ENVELOPE_PLAIN = 0x01; const BACKUP_ENVELOPE_GZIP = 0x02; const BACKUP_ENVELOPE_STREAM = 0x03; +/** + * Version 4 adds two things version 3 did not have, and changes nothing else. + * + * A key id: the first twelve hex characters of the key's SHA-256, so a reader + * holding several keys (`BACKUP_ENCRYPTION_PREVIOUS_KEYS` during a rotation) + * knows which one opens an object instead of trying each. It is a + * fingerprint, not a name the operator has to keep in step. + * + * Associated data: the object's key in the bucket, which names the account + * and the night. Before this, an object copied to another account's key in + * the bucket, or to another date, decrypted as if it belonged there; now it + * does not open at all. + */ +const BACKUP_ENVELOPE_KEYED = 0x04; +const OFFHOST_AAD_PREFIX = "healthlog/offhost-backup/v4|"; const MAGIC = "HLBK"; /** magic(4) + version(1). Where the per-version body begins. */ const PREAMBLE_LENGTH = 5; +/** The off-host keys a reader may use: the current one, then the retired. */ +export interface OffhostKeyRing { + active: Buffer; + previous: Buffer[]; +} + +/** The key ring of a loaded configuration. */ +export function offhostKeyRing(cfg: OffhostBackupConfig): OffhostKeyRing { + return { active: cfg.encryptionKey, previous: cfg.previousEncryptionKeys }; +} + +function asKeyRing(key: Buffer | OffhostKeyRing): OffhostKeyRing { + return Buffer.isBuffer(key) ? { active: key, previous: [] } : key; +} + +/** The id a version-4 object records for the key it was written under. */ +export function offhostKeyId(key: Buffer): string { + return createHash("sha256").update(key).digest("hex").slice(0, 12); +} + +function offhostAad(objectKey: string): Buffer { + return Buffer.from(`${OFFHOST_AAD_PREFIX}${objectKey}`, "utf8"); +} + +/** + * Open the gzip bytes of a version-4 object: find its key by id, verify the + * tag with the object key as associated data. + */ +function openKeyed( + buf: Buffer, + ring: OffhostKeyRing, + objectKey: string | undefined, +): Buffer { + if (objectKey === undefined) { + throw new Error( + "This backup object is bound to its key in the bucket; pass the object key to open it", + ); + } + const idLen = buf[PREAMBLE_LENGTH]; + const keyId = buf + .subarray(PREAMBLE_LENGTH + 1, PREAMBLE_LENGTH + 1 + idLen) + .toString("latin1"); + const key = [ring.active, ...ring.previous].find( + (candidate) => offhostKeyId(candidate) === keyId, + ); + if (!key) { + throw new Error( + `Backup object was written under off-host key ${keyId}, which is neither BACKUP_ENCRYPTION_KEY nor in BACKUP_ENCRYPTION_PREVIOUS_KEYS`, + ); + } + return decryptRawStream( + buf.subarray(PREAMBLE_LENGTH + 1 + idLen), + key, + offhostAad(objectKey), + ); +} + +/** + * Try each key of the ring on an object with no key id (versions 1 to 3), + * current first. GCM refuses a wrong key outright, so the first key that + * opens it is the one it was written under. + */ +function withEachKey(ring: OffhostKeyRing, open: (key: Buffer) => T): T { + let lastError: unknown = null; + for (const key of [ring.active, ...ring.previous]) { + try { + return open(key); + } catch (err) { + lastError = err; + } + } + throw lastError ?? new Error("No off-host key configured"); +} + +function isKnownVersion(version: number): boolean { + return ( + version === BACKUP_ENVELOPE_PLAIN || + version === BACKUP_ENVELOPE_GZIP || + version === BACKUP_ENVELOPE_STREAM || + version === BACKUP_ENVELOPE_KEYED + ); +} + /** * Write a whole JSON string as a version-2 object. * @@ -163,23 +284,36 @@ export function encryptBackup(plaintext: string, key: Buffer): Buffer { return Buffer.concat([header, iv, tag, ct]); } -export function decryptBackup(buf: Buffer, key: Buffer): string { +export function decryptBackup( + buf: Buffer, + keys: Buffer | OffhostKeyRing, + objectKey?: string, +): string { const magic = buf.subarray(0, 4).toString("binary"); const version = buf[PREAMBLE_LENGTH - 1]; - if ( - magic !== MAGIC || - (version !== BACKUP_ENVELOPE_PLAIN && - version !== BACKUP_ENVELOPE_GZIP && - version !== BACKUP_ENVELOPE_STREAM) - ) { + if (magic !== MAGIC || !isKnownVersion(version)) { throw new Error("Invalid backup envelope (bad magic or version)"); } + const ring = asKeyRing(keys); + if (version === BACKUP_ENVELOPE_KEYED) { + return gunzipSync(openKeyed(buf, ring, objectKey)).toString("utf8"); + } if (version === BACKUP_ENVELOPE_STREAM) { // iv | ciphertext | tag, exactly what the streaming writer emits and what // the shared reader verifies whole before it hands back a byte. - const plaintext = decryptRawStream(buf.subarray(PREAMBLE_LENGTH), key); + const plaintext = withEachKey(ring, (key) => + decryptRawStream(buf.subarray(PREAMBLE_LENGTH), key), + ); return gunzipSync(plaintext).toString("utf8"); } + const plaintext = withEachKey(ring, (key) => openLeadingTag(buf, key)); + return version === BACKUP_ENVELOPE_GZIP + ? gunzipSync(plaintext).toString("utf8") + : plaintext.toString("utf8"); +} + +/** Versions 1 and 2: iv | tag | ciphertext. */ +function openLeadingTag(buf: Buffer, key: Buffer): Buffer { const iv = buf.subarray(PREAMBLE_LENGTH, PREAMBLE_LENGTH + IV_LENGTH); const tag = buf.subarray( PREAMBLE_LENGTH + IV_LENGTH, @@ -188,10 +322,7 @@ export function decryptBackup(buf: Buffer, key: Buffer): string { const ct = buf.subarray(PREAMBLE_LENGTH + IV_LENGTH + TAG_LENGTH); const dec = createDecipheriv(ALGORITHM, key, iv); dec.setAuthTag(tag); - const plaintext = Buffer.concat([dec.update(ct), dec.final()]); - return version === BACKUP_ENVELOPE_GZIP - ? gunzipSync(plaintext).toString("utf8") - : plaintext.toString("utf8"); + return Buffer.concat([dec.update(ct), dec.final()]); } /** @@ -204,31 +335,24 @@ export function decryptBackup(buf: Buffer, key: Buffer): string { */ export function openBackupObject( buf: Buffer, - key: Buffer, + keys: Buffer | OffhostKeyRing, + objectKey?: string, ): () => AsyncIterable { const magic = buf.subarray(0, 4).toString("binary"); const version = buf[PREAMBLE_LENGTH - 1]; - if ( - magic !== MAGIC || - (version !== BACKUP_ENVELOPE_PLAIN && - version !== BACKUP_ENVELOPE_GZIP && - version !== BACKUP_ENVELOPE_STREAM) - ) { + if (magic !== MAGIC || !isKnownVersion(version)) { throw new Error("Invalid backup envelope (bad magic or version)"); } + const ring = asKeyRing(keys); let plaintext: Buffer; - if (version === BACKUP_ENVELOPE_STREAM) { - plaintext = decryptRawStream(buf.subarray(PREAMBLE_LENGTH), key); - } else { - const iv = buf.subarray(PREAMBLE_LENGTH, PREAMBLE_LENGTH + IV_LENGTH); - const tag = buf.subarray( - PREAMBLE_LENGTH + IV_LENGTH, - PREAMBLE_LENGTH + IV_LENGTH + TAG_LENGTH, + if (version === BACKUP_ENVELOPE_KEYED) { + plaintext = openKeyed(buf, ring, objectKey); + } else if (version === BACKUP_ENVELOPE_STREAM) { + plaintext = withEachKey(ring, (key) => + decryptRawStream(buf.subarray(PREAMBLE_LENGTH), key), ); - const ct = buf.subarray(PREAMBLE_LENGTH + IV_LENGTH + TAG_LENGTH); - const dec = createDecipheriv(ALGORITHM, key, iv); - dec.setAuthTag(tag); - plaintext = Buffer.concat([dec.update(ct), dec.final()]); + } else { + plaintext = withEachKey(ring, (key) => openLeadingTag(buf, key)); } if (version === BACKUP_ENVELOPE_PLAIN) { return () => Readable.from([plaintext]); @@ -245,6 +369,9 @@ export function openBackupObject( const UPLOAD_PART_BYTES = 8 * 1024 * 1024; const UPLOAD_CONCURRENCY = 2; +/** Pages of 1 000 keys one listing may walk: 200 000 objects. */ +const MAX_LIST_PAGES = 200; + /** * The largest object one multipart upload can carry: S3 and every compatible * target cap a multipart upload at 10 000 parts. @@ -303,12 +430,15 @@ export class OffhostBackupTooLargeError extends Error { */ function createEnvelopeStream( key: Buffer, + objectKey: string, limitBytes: number, ): { stream: Transform; bytes: () => number } { - const encryptor = createRawStreamEncryptor(key); + const encryptor = createRawStreamEncryptor(key, offhostAad(objectKey)); + const keyId = Buffer.from(offhostKeyId(key), "latin1"); const header = Buffer.concat([ Buffer.from(MAGIC, "binary"), - Buffer.from([BACKUP_ENVELOPE_STREAM]), + Buffer.from([BACKUP_ENVELOPE_KEYED, keyId.byteLength]), + keyId, encryptor.iv, ]); let written = 0; @@ -395,6 +525,7 @@ export async function uploadEncryptedBackup( const gzip = createGzip(); const { stream: envelope, bytes } = createEnvelopeStream( encryptionKey, + objectKey, limitBytes, ); @@ -487,6 +618,24 @@ export interface S3Like { prefix: string, ): Promise>; deleteObject(key: string): Promise; + /** + * The bucket's lifecycle rule, as far as the credential may read it. + * Optional so a test double that has no opinion leaves it out. + */ + getLifecycle?(): Promise; +} + +/** + * Whether the bucket retires old copies by itself. + * + * `configured` names the shortest expiry of an enabled rule; `missing` means + * the bucket answered that it has no rule; `unknown` means it would not say + * (a credential without `s3:GetLifecycleConfiguration`, a target that does not + * implement the call, a network failure) and says nothing either way. + */ +export interface OffhostLifecycle { + state: "configured" | "missing" | "unknown"; + expirationDays: number | null; } export async function getS3Client(cfg: OffhostBackupConfig): Promise { @@ -568,22 +717,109 @@ export async function getS3Client(cfg: OffhostBackupConfig): Promise { } }, listObjects: async (prefix) => { - const out = await client.send( - new mod.ListObjectsV2Command({ Bucket: cfg.bucket, Prefix: prefix }), + // Every page, not the first. A bucket answers at most 1 000 keys per + // call, in key order, so the first page of a bucket with a few accounts + // and a month of retention holds only the OLDEST dates: the restore + // drill read an old object and called the chain stale, and a purge + // would have missed every newer copy. Bounded, so a bucket that holds + // far more than this job put there cannot keep the loop going forever. + const found: Array<{ key: string; lastModified?: Date }> = []; + let token: string | undefined; + for (let page = 0; page < MAX_LIST_PAGES; page++) { + const out = await client.send( + new mod.ListObjectsV2Command({ + Bucket: cfg.bucket, + Prefix: prefix, + ContinuationToken: token, + }), + ); + for (const c of out.Contents ?? []) { + found.push({ key: c.Key ?? "", lastModified: c.LastModified }); + } + if (!out.IsTruncated || !out.NextContinuationToken) return found; + token = out.NextContinuationToken; + } + throw new Error( + `Bucket listing under "${prefix}" did not end within ${MAX_LIST_PAGES} pages`, ); - return (out.Contents ?? []).map((c) => ({ - key: c.Key ?? "", - lastModified: c.LastModified, - })); }, deleteObject: async (key) => { await client.send( new mod.DeleteObjectCommand({ Bucket: cfg.bucket, Key: key }), ); }, + getLifecycle: async () => { + try { + const out = await client.send( + new mod.GetBucketLifecycleConfigurationCommand({ + Bucket: cfg.bucket, + }), + { abortSignal: AbortSignal.timeout(5_000) }, + ); + const days = (out.Rules ?? []) + .filter((rule) => rule.Status === "Enabled") + .map((rule) => rule.Expiration?.Days) + .filter((d): d is number => typeof d === "number" && d > 0); + return days.length > 0 + ? { state: "configured", expirationDays: Math.min(...days) } + : { state: "missing", expirationDays: null }; + } catch (err) { + const name = (err as { name?: string; Code?: string }).name; + const code = (err as { Code?: string }).Code; + if ( + name === "NoSuchLifecycleConfiguration" || + code === "NoSuchLifecycleConfiguration" + ) { + return { state: "missing", expirationDays: null }; + } + return { state: "unknown", expirationDays: null }; + } + }, }; } +const LIFECYCLE_CACHE_MS = 10 * 60 * 1000; +let lifecycleCache: { at: number; value: OffhostLifecycle } | null = null; + +/** + * The bucket's lifecycle rule for the admin card, cached for ten minutes so a + * page render does not become a bucket call. Never throws. + */ +export async function probeOffhostLifecycle( + s3Override?: S3Like, + now: number = Date.now(), +): Promise { + if ( + !s3Override && + lifecycleCache && + now - lifecycleCache.at < LIFECYCLE_CACHE_MS + ) { + return lifecycleCache.value; + } + const cfg = loadOffhostConfigSafe(); + if (!cfg) return { state: "unknown", expirationDays: null }; + let value: OffhostLifecycle; + try { + const s3 = s3Override ?? (await getS3Client(cfg)); + value = s3.getLifecycle + ? await s3.getLifecycle() + : { state: "unknown", expirationDays: null }; + } catch { + value = { state: "unknown", expirationDays: null }; + } + if (!s3Override) lifecycleCache = { at: now, value }; + return value; +} + +/** `loadOffhostConfig`, answering null for a malformed key instead of throwing. */ +function loadOffhostConfigSafe(): OffhostBackupConfig | null { + try { + return loadOffhostConfig(); + } catch { + return null; + } +} + interface BackupRunReport { config: { endpoint: string; bucket: string; region: string }; uploaded: number; @@ -622,6 +858,30 @@ export interface RunOffhostBackupOptions extends UploadBackupOptions { shouldStop?: () => boolean; } +/** + * Note, per key id, that an object needing it went into the bucket now. Never + * fails the account: the object is in the bucket whatever this row says. + */ +async function recordOffhostKeyUse( + prisma: PrismaClient, + keyIds: readonly string[], + at: Date, +): Promise { + for (const keyId of keyIds) { + try { + await prisma.offhostBackupKeyUse.upsert({ + where: { keyId }, + create: { keyId, firstWrittenAt: at, lastWrittenAt: at }, + update: { lastWrittenAt: at }, + }); + } catch (err) { + getEvent()?.addWarning( + `offhost-backup key-use write failed: ${(err as Error).message?.slice(0, 200)}`, + ); + } + } +} + export async function runOffhostBackup( prisma: PrismaClient, s3Override?: S3Like, @@ -669,24 +929,54 @@ export async function runOffhostBackup( break; } let objectBytes: number | null = null; + const objectKey = `${dateKey}/user-${user.id}.json.enc`; + const accountStartedAt = new Date(); + // The key ids the object's content needs, read as it is written: the + // admin encryption view uses them to say how long a retired key is still + // needed for what is in the bucket. + const keyScanner = createBackupKeyIdTextScanner(); try { objectBytes = await uploadEncryptedBackup( s3, - `${dateKey}/user-${user.id}.json.enc`, + objectKey, cfg.encryptionKey, // The same writer the weekly in-database pass uses. The payload // builder was always shared; everything after it was not, which is why // this job kept dying on a record the weekly one had learned to // survive. (write) => - streamFullBackupJson(prisma, user.id, write, { - purpose: "disaster-recovery", - exportedAt: now, - }), + streamFullBackupJson( + prisma, + user.id, + async (chunk) => { + keyScanner.feed(chunk); + await write(chunk); + }, + { + purpose: "disaster-recovery", + exportedAt: now, + }, + ), options, ); + // The account was deleted or wiped while its copy was being written. + // The purge that request started may already have run, so the copy + // this run just put there is removed here rather than left behind. + const purgedMeanwhile = + (await prisma.offhostPurgeRequest.count({ + where: { + subjectId: user.id, + requestedAt: { gte: accountStartedAt }, + }, + })) > 0 || (await prisma.user.count({ where: { id: user.id } })) === 0; + if (purgedMeanwhile) { + await s3.deleteObject(objectKey); + objectBytes = null; + continue; + } largestObjectBytes = Math.max(largestObjectBytes, objectBytes); uploaded++; + await recordOffhostKeyUse(prisma, keyScanner.keyIds(), new Date()); } catch (err) { failed++; if (err instanceof OffhostBackupTooLargeError) oversized++; diff --git a/src/lib/jobs/offhost-purge.ts b/src/lib/jobs/offhost-purge.ts new file mode 100644 index 000000000..15585fb03 --- /dev/null +++ b/src/lib/jobs/offhost-purge.ts @@ -0,0 +1,260 @@ +/** + * Off-host copies leave the bucket with the account. + * + * The nightly job writes `/user-.json.enc` for every account, + * and until v1.39.3 nothing ever removed one: deleting an account, or wiping + * its data, took the rows in this database and left every nightly copy of the + * record in the operator's bucket for as long as its lifecycle rule kept them, + * or forever where no rule was set. A confirmation that says "permanently + * delete" has to mean the off-host copies too. + * + * How. The deletion writes an `OffhostPurgeRequest` in its own transaction, so + * the request exists exactly when the deletion committed. This job then lists + * the bucket once, deletes every object of every pending account, and removes + * the request when nothing of the account is left. A request the bucket + * refuses (a credential without DeleteObject, an outage) stays, counts its + * attempts and keeps the bucket's answer, which the admin off-host card shows; + * the job runs again every night and after every nightly upload, so a late + * copy the upload wrote for an account deleted while it ran is caught too. + * + * The admin wipe of every account writes no request, on purpose: it is the + * operator clearing their own host, the bucket is theirs, and its copies are + * the documented way back from a wipe pressed by mistake. The health-check + * probes and anything else in the bucket that is not an account object is + * never touched. + */ +import type { Job } from "pg-boss"; + +import type { Prisma, PrismaClient } from "@/generated/prisma/client"; +import { auditLog } from "@/lib/auth/audit"; +import { getGlobalBoss } from "@/lib/jobs/boss-instance"; +import { jobDone, jobFailed, type JobOutcome } from "@/lib/jobs/job-outcome"; +import { + getS3Client, + loadOffhostConfig, + offhostBackupConfigured, + type S3Like, +} from "@/lib/jobs/offhost-backup"; +import { reportWorkerError } from "@/lib/jobs/report-worker-error"; +import { getWorkerPrisma } from "@/lib/jobs/reminder/shared"; +import { withBackgroundEvent } from "@/lib/logging/background"; + +export const OFFHOST_PURGE_QUEUE = "offhost-backup-purge"; +/** 03:40, after the nightly upload (02:30) has had its hour. */ +export const OFFHOST_PURGE_CRON = "40 3 * * *"; + +export type OffhostPurgeReason = + "account_deleted" | "data_wiped" | "managed_profile_deleted"; + +/** An account object: `/user-.json.enc`. */ +const ACCOUNT_OBJECT = /^\d{4}-\d{2}-\d{2}\/user-(.+)\.json\.enc$/; + +/** Requests one run works through; the next run takes the rest. */ +const REQUESTS_PER_RUN = 200; + +/** Deletes one run may issue, so a huge bucket cannot run the job forever. */ +const MAX_DELETES_PER_RUN = 50_000; + +type Db = Pick; + +/** + * Record that `subjectId`'s off-host copies have to go. Call it with the + * transaction that deletes the account or wipes its data, so the request and + * the deletion commit together. A host without off-host backup configured has + * nothing in any bucket this could reach, and records nothing. + */ +export async function requestOffhostPurge( + db: Db, + subjectId: string, + reason: OffhostPurgeReason, +): Promise { + if (!offhostBackupConfigured()) return false; + await db.offhostPurgeRequest.create({ + data: { subjectId, reason }, + select: { id: true }, + }); + return true; +} + +/** + * Wake the purge job now rather than at its nightly slot. Best effort: the + * request row is what guarantees the purge, and the cron picks it up if the + * queue is not reachable from here. + */ +export async function kickOffhostPurge(): Promise { + try { + await getGlobalBoss()?.send( + OFFHOST_PURGE_QUEUE, + {}, + { + singletonKey: OFFHOST_PURGE_QUEUE, + retryLimit: 3, + retryDelay: 300, + }, + ); + } catch { + // The cron is the backstop. + } +} + +export interface OffhostPurgeReport { + pending: number; + completed: number; + failed: number; + objectsDeleted: number; +} + +function trimError(err: unknown): string { + return (err instanceof Error ? err.message : String(err)).slice(0, 300); +} + +/** + * Work through the pending requests: list the bucket once, delete every + * object of every pending subject, remove each request whose objects are all + * gone. Answers what it did. + */ +export async function processOffhostPurges( + prisma: PrismaClient, + s3Override?: S3Like, +): Promise { + const requests = await prisma.offhostPurgeRequest.findMany({ + orderBy: { requestedAt: "asc" }, + take: REQUESTS_PER_RUN, + }); + const report: OffhostPurgeReport = { + pending: requests.length, + completed: 0, + failed: 0, + objectsDeleted: 0, + }; + if (requests.length === 0) return report; + + const cfg = loadOffhostConfig(); + if (!cfg) { + // Configuration removed after the requests were written: leave them, an + // operator who puts it back gets the purge. + return report; + } + + const now = new Date(); + const markFailed = async (ids: string[], err: unknown) => { + await prisma.offhostPurgeRequest.updateMany({ + where: { id: { in: ids } }, + data: { + attempts: { increment: 1 }, + lastAttemptAt: now, + lastFailure: trimError(err), + }, + }); + report.failed += ids.length; + }; + + let objects: Array<{ key: string }>; + let s3: S3Like; + try { + s3 = s3Override ?? (await getS3Client(cfg)); + objects = await s3.listObjects(""); + } catch (err) { + await markFailed( + requests.map((r) => r.id), + err, + ); + return report; + } + + const subjects = new Set(requests.map((r) => r.subjectId)); + const bySubject = new Map(); + for (const { key } of objects) { + const match = ACCOUNT_OBJECT.exec(key); + if (!match) continue; + const subject = match[1]; + if (!subjects.has(subject)) continue; + const list = bySubject.get(subject) ?? []; + list.push(key); + bySubject.set(subject, list); + } + + // One subject at a time, so a refusal part-way fails only the subjects it + // reached, and a request is removed only when its own objects are gone. + const failedSubjects = new Map(); + let deletes = 0; + for (const [subject, keys] of bySubject) { + for (const key of keys) { + if (deletes >= MAX_DELETES_PER_RUN) { + failedSubjects.set( + subject, + new Error("Deletion budget for one run reached; continues next run"), + ); + break; + } + try { + await s3.deleteObject(key); + deletes++; + report.objectsDeleted++; + } catch (err) { + failedSubjects.set(subject, err); + break; + } + } + } + + for (const request of requests) { + const failure = failedSubjects.get(request.subjectId); + if (failure) { + await markFailed([request.id], failure); + continue; + } + const deleted = bySubject.get(request.subjectId)?.length ?? 0; + await prisma.offhostPurgeRequest.delete({ where: { id: request.id } }); + report.completed++; + // The account a deleted subject named is gone, and so is its audit + // history; the receipt names the reason and the count, not the account. + // A wiped account still exists and keeps the receipt as its own. + const accountRemains = + request.reason === "data_wiped" && + (await prisma.user.count({ where: { id: request.subjectId } })) > 0; + await auditLog("offhost.backup.purged", { + userId: accountRemains ? request.subjectId : null, + actorUserId: null, + details: { + reason: request.reason, + objectsDeleted: deleted, + attempts: request.attempts + 1, + }, + }); + } + return report; +} + +export async function handleOffhostPurge( + jobs: Job[], +): Promise { + void jobs; + return withBackgroundEvent("job.offhost_purge", async (evt) => { + try { + const report = await processOffhostPurges(getWorkerPrisma()); + evt.addMeta("offhost_purge_pending", report.pending); + evt.addMeta("offhost_purge_completed", report.completed); + evt.addMeta("offhost_purge_failed", report.failed); + evt.addMeta("offhost_purge_objects_deleted", report.objectsDeleted); + if (report.failed > 0) { + await reportWorkerError( + OFFHOST_PURGE_QUEUE, + new Error( + `${report.failed} off-host deletion request(s) could not be completed; they stay pending and are retried nightly. Check that the bucket credential allows DeleteObject.`, + ), + ); + } + return jobDone({ + offhost_purge_pending: report.pending, + offhost_purge_completed: report.completed, + offhost_purge_failed: report.failed, + offhost_purge_objects_deleted: report.objectsDeleted, + }); + } catch (err) { + evt.addWarning(`offhost-purge failed: ${err}`); + await reportWorkerError(OFFHOST_PURGE_QUEUE, err); + return jobFailed("offhost purge failed", err); + } + }); +} diff --git a/src/lib/jobs/reminder/register-maintenance.ts b/src/lib/jobs/reminder/register-maintenance.ts index 0aa69e355..149e91d45 100644 --- a/src/lib/jobs/reminder/register-maintenance.ts +++ b/src/lib/jobs/reminder/register-maintenance.ts @@ -67,6 +67,11 @@ import { enqueueBootTimeIntakeSlotDedup, type IntakeSlotDedupPayload, } from "@/lib/medications/intake-slot-dedup"; +import { + handleOffhostPurge, + OFFHOST_PURGE_CRON, + OFFHOST_PURGE_QUEUE, +} from "@/lib/jobs/offhost-purge"; import { handleRestoreDrill, RESTORE_DRILL_CRON, @@ -353,6 +358,10 @@ const allQueues = [ STEP_UP_ELEVATION_CLEANUP_QUEUE, OFFHOST_BACKUP_QUEUE, RESTORE_DRILL_QUEUE, + // v1.39.3 — deleting an account or wiping its data takes its off-host + // copies out of the bucket. Without this entry the requests would be + // written and never worked. + OFFHOST_PURGE_QUEUE, // v1.39.1 — an admin's restore of a stored backup, run off the request, and // the boot sweep that re-queues one a stopped worker left running. Without // this entry every restore request would queue a job nobody works. @@ -505,6 +514,9 @@ const schedules: ScheduleEntry[] = [ ], [OFFHOST_BACKUP_QUEUE, OFFHOST_BACKUP_CRON], [RESTORE_DRILL_QUEUE, RESTORE_DRILL_CRON], + // Nightly backstop for the purge the deletion kicks straight away: a + // request the bucket refused, or one written while the queue was down. + [OFFHOST_PURGE_QUEUE, OFFHOST_PURGE_CRON, cronIsTheRetry], [HOST_METRIC_QUEUE, HOST_METRIC_CRON, cronIsTheRetry], [FEEDBACK_AGGREGATOR_QUEUE, FEEDBACK_AGGREGATOR_CRON, cronIsTheRetry], // v1.4.37 — hourly geo backfill. The helper is idempotent + capped @@ -747,6 +759,12 @@ export async function registerMaintenanceQueues( { localConcurrency: 1 }, handleRestoreDrill, ); + await createAndWork( + boss, + OFFHOST_PURGE_QUEUE, + { localConcurrency: 1 }, + handleOffhostPurge, + ); // One restore at a time per worker: each holds a transaction over a whole // account and a batch of readings in memory, and two at once on a small host // is how a restore that fits alone would run out of heap. diff --git a/src/lib/jobs/restore-drill.ts b/src/lib/jobs/restore-drill.ts index 50450c630..4571af414 100644 --- a/src/lib/jobs/restore-drill.ts +++ b/src/lib/jobs/restore-drill.ts @@ -23,8 +23,14 @@ */ import type { Job } from "pg-boss"; import { BackupJsonError, scanBackupJson } from "@/lib/export/backup-json-scan"; +import { + assessBackupKeys, + BackupKeyIdCollector, + describeBackupKeyProblem, +} from "@/lib/export/backup-key-ids"; import { openBackupObject, + offhostKeyRing, getS3Client, loadOffhostConfig, OffhostBackupNotConfiguredError, @@ -54,6 +60,8 @@ export interface RestoreDrillReport { stale: boolean; ciphertextBytes: number; plaintextBytes: number; + /** Application key ids the object's inner ciphertext needs, each proven to open. */ + innerKeyIds: string[]; recordCounts: { measurements: number; medications: number; @@ -107,7 +115,7 @@ export async function runRestoreDrill( // Read as a stream, never as one string: the JSON of a large record is // longer than any string V8 can hold (#1031). The bulk tables are counted, // not kept. - const source = openBackupObject(ciphertext, cfg.encryptionKey); + const source = openBackupObject(ciphertext, offhostKeyRing(cfg), objectKey); let plaintextBytes = 0; async function* counted() { for await (const chunk of source()) { @@ -115,10 +123,17 @@ export async function runRestoreDrill( yield chunk; } } + // The envelope opening proves the off-host key. It proves nothing about + // the application keys the ciphertext INSIDE was written under, and that + // is the half a key rotation breaks: a restore of this object writes those + // values back verbatim. So every inner key id is collected, and one value + // per key is actually decrypted. + const keys = new BackupKeyIdCollector(); let scanned; try { scanned = await scanBackupJson(counted(), { streamKeys: new Set(["measurements", "intakeEvents", "moodEntries"]), + onElement: (key, element) => keys.visit(element, key), }); } catch (err) { if (err instanceof BackupJsonError) { @@ -140,6 +155,17 @@ export async function runRestoreDrill( ); } + keys.visit(payload); + const keyVerdict = assessBackupKeys(keys, { + ignoreSections: new Set(["appSettings"]), + }); + const keyProblem = describeBackupKeyProblem(keyVerdict); + if (keyProblem) { + throw new Error( + `Restore drill: backup object "${objectKey}" opens, but its content could not be restored on this server. ${keyProblem}`, + ); + } + const ageDays = Math.floor( (now.getTime() - Date.parse(`${dateKey}T00:00:00Z`)) / (24 * 60 * 60 * 1000), @@ -152,6 +178,7 @@ export async function runRestoreDrill( stale: ageDays > MAX_BACKUP_AGE_DAYS, ciphertextBytes: ciphertext.length, plaintextBytes, + innerKeyIds: keyVerdict.keyIds, recordCounts: { measurements: streamedCounts.measurements, medications: countArray(payload.medications), @@ -193,6 +220,7 @@ export async function handleRestoreDrill( report.recordCounts.moodEntries, ); evt.addMeta("restore_drill_stale", report.stale); + evt.addMeta("restore_drill_inner_key_ids", report.innerKeyIds.join(",")); if (report.stale) { await reportWorkerError( RESTORE_DRILL_QUEUE, diff --git a/src/lib/jobs/subsystem-surface.ts b/src/lib/jobs/subsystem-surface.ts index bbfeffc72..cdf8b17d4 100644 --- a/src/lib/jobs/subsystem-surface.ts +++ b/src/lib/jobs/subsystem-surface.ts @@ -101,6 +101,7 @@ const subsystemSurface = { "step-up-elevation-cleanup": { audience: "system" }, "data-backup-offhost": { audience: "system" }, "data-restore-drill": { audience: "system" }, + "offhost-backup-purge": { audience: "system" }, "backup-restore": { audience: "system" }, "host-metric-sample": { audience: "system" }, "feedback-aggregator": { audience: "system" }, diff --git a/src/lib/managed-profiles/lifecycle.ts b/src/lib/managed-profiles/lifecycle.ts index 44ddac8ac..1160825e5 100644 --- a/src/lib/managed-profiles/lifecycle.ts +++ b/src/lib/managed-profiles/lifecycle.ts @@ -1,3 +1,7 @@ +import { + kickOffhostPurge, + requestOffhostPurge, +} from "@/lib/jobs/offhost-purge"; import { prisma } from "@/lib/db"; import { auditLog } from "@/lib/auth/audit"; import type { Prisma } from "@/generated/prisma/client"; @@ -209,9 +213,11 @@ export async function deleteManagedProfile(input: { details: { profileId: profile.id }, client: tx, }); + await requestOffhostPurge(tx, profile.id, "managed_profile_deleted"); await tx.user.delete({ where: { id: profile.id } }); }); }); + await kickOffhostPurge(); } /** diff --git a/src/lib/multipart/__tests__/stream-to-disk.test.ts b/src/lib/multipart/__tests__/stream-to-disk.test.ts index 42f8329c3..2d5386bd0 100644 --- a/src/lib/multipart/__tests__/stream-to-disk.test.ts +++ b/src/lib/multipart/__tests__/stream-to-disk.test.ts @@ -1,5 +1,7 @@ import { describe, expect, it } from "vitest"; -import { readFileSync, unlinkSync } from "node:fs"; +import { mkdtempSync, readdirSync, readFileSync, unlinkSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; import { createHash } from "node:crypto"; import { deflateRawSync } from "node:zlib"; @@ -112,6 +114,29 @@ describe("streamMultipartToDisk", () => { ).rejects.toThrow(/size cap/); }); + // What was written before the refusal is the person's export in plain + // text; it must not stay behind in the temp directory. + it("removes the partial file when the upload is refused part-way", async () => { + const dir = mkdtempSync(join(tmpdir(), "healthlog-stream-")); + const body = buildMultipartBody("export.zip", Buffer.alloc(4096, 0x41)); + const chunked = new ReadableStream({ + start(controller) { + for (let i = 0; i < body.length; i += 256) { + controller.enqueue(new Uint8Array(body.subarray(i, i + 256))); + } + controller.close(); + }, + }); + await expect( + streamMultipartToDisk( + chunked, + `multipart/form-data; boundary=${BOUNDARY}`, + { maxBytes: 1000, fieldName: "file", tmpDir: dir }, + ), + ).rejects.toThrow(/size cap/); + expect(readdirSync(dir)).toEqual([]); + }); + it("throws when the named field is missing", async () => { const body = buildMultipartBody("export.zip", Buffer.from("x"), { something: "else", diff --git a/src/lib/multipart/stream-to-disk.ts b/src/lib/multipart/stream-to-disk.ts index 1bc734df9..fcc8fe650 100644 --- a/src/lib/multipart/stream-to-disk.ts +++ b/src/lib/multipart/stream-to-disk.ts @@ -17,6 +17,7 @@ */ import { createHash, randomUUID } from "node:crypto"; import { createWriteStream } from "node:fs"; +import { rm } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { Writable } from "node:stream"; @@ -293,22 +294,41 @@ export async function streamMultipartToDisk( } }; - while (!streamDone) { - const { value, done } = await reader.read(); - if (done) { - streamDone = true; - break; + try { + while (!streamDone) { + const { value, done } = await reader.read(); + if (done) { + streamDone = true; + break; + } + buffer = Buffer.concat([buffer, Buffer.from(value)]); + await processBuffer(); } - buffer = Buffer.concat([buffer, Buffer.from(value)]); await processBuffer(); - } - await processBuffer(); - await closeFileSink(); + await closeFileSink(); - if (bytesWritten === 0) { - throw new Error( - `Multipart body did not include a '${opts.fieldName}' file field`, - ); + if (bytesWritten === 0) { + throw new Error( + `Multipart body did not include a '${opts.fieldName}' file field`, + ); + } + } catch (err) { + // Whatever was written so far is the user's plaintext health export, + // and it must not outlive a refused upload in the shared temp + // directory: the size cap, a malformed body and a dropped connection + // all end here. The reader is cancelled so the rest of the body is not + // read for nothing. + await reader.cancel().catch(() => {}); + const sink = fileSink as Writable | null; + if (sink && !sink.closed) { + fileSinkOpen = false; + await new Promise((resolve) => { + sink.once("close", () => resolve()); + sink.destroy(); + }); + } + await rm(filePath, { force: true }).catch(() => {}); + throw err; } return { diff --git a/src/lib/openapi/error-codes.ts b/src/lib/openapi/error-codes.ts index 87b7aabf5..8a7b57fb0 100644 --- a/src/lib/openapi/error-codes.ts +++ b/src/lib/openapi/error-codes.ts @@ -69,6 +69,8 @@ export const ERROR_CODE_CATALOGUE: Readonly> = "auth.token.invalid", ], backup: [ + "backup.foreign_reference", + "backup.key.missing", "backup.payload.undecryptable", "backup.restore.active", "backup.section.missing", @@ -185,7 +187,7 @@ export const ERROR_CODE_CATALOGUE: Readonly> = "illness.episode.parent-not-found", "illness.insights.invalid", ], - import: ["import.write_failed"], + import: ["import.apple_health.busy", "import.write_failed"], ingest: ["ingest.rate_limited"], insights: [ "insights.generate.budgetExceeded", diff --git a/src/lib/openapi/routes/admin-backups.ts b/src/lib/openapi/routes/admin-backups.ts index 44edd3fdc..0c4bebf6b 100644 --- a/src/lib/openapi/routes/admin-backups.ts +++ b/src/lib/openapi/routes/admin-backups.ts @@ -86,15 +86,16 @@ const restoreFailure = z .object({ code: z.string().meta({ description: - "Stable reason: `backup_not_found`, `backup_changed`, `backup.payload.undecryptable`, `schema_invalid`, `incompatible_schema_version`, `owner_mismatch`, `owner_not_found`, `backup.section.missing`, `document_ciphertext_missing`, `time_budget`, `transaction_failed`, `interrupted`, `not_started`, `enqueue_failed`, `failed_after_commit`, `unexpected`. Every code but `failed_after_commit` means the account was not changed.", + "Stable reason: `backup_not_found`, `backup_changed`, `backup.payload.undecryptable`, `schema_invalid`, `incompatible_schema_version`, `owner_mismatch`, `owner_not_found`, `backup.section.missing`, `document_ciphertext_missing`, `backup.key.missing`, `backup.foreign_reference`, `time_budget`, `transaction_failed`, `interrupted`, `not_started`, `enqueue_failed`, `failed_after_commit`, `unexpected`. Every code but `failed_after_commit` means the account was not changed.", }), message: z.string(), sections: z.array(z.string()).optional(), + keyIds: z.array(z.string()).optional(), }) .meta({ id: "AdminBackupRestoreFailure", description: - "Why the restore did not happen, in a sentence an operator can act on. `sections` names the missing sections for `backup.section.missing`.", + "Why the restore did not happen, in a sentence an operator can act on. `sections` names the missing sections for `backup.section.missing`; `keyIds` names the encryption key ids the file needs and this server cannot open it with, for `backup.key.missing`.", }); const restoreJob = z diff --git a/src/lib/openapi/routes/import.ts b/src/lib/openapi/routes/import.ts index 04cc258eb..82c8431dd 100644 --- a/src/lib/openapi/routes/import.ts +++ b/src/lib/openapi/routes/import.ts @@ -324,6 +324,11 @@ export const importPaths: NonNullable = { content: { "application/json": { schema: errorEnvelope } }, }, ...stdResponses, + "409": { + description: + "An import for this account is already queued or running, and these bytes are not the ones it carries (the same bytes resolve to that job as a 202). `meta.errorCode` is `import.apple_health.busy` and `meta.jobId` names the running job; poll it, then upload again. The upload was discarded.", + content: { "application/json": { schema: errorEnvelope } }, + }, "422": { description: "The multipart body could not be streamed to disk — no `file` part, a malformed boundary, or the size cap tripped mid-stream. The message names the failure.", @@ -335,7 +340,7 @@ export const importPaths: NonNullable = { }, "503": { description: - "No background worker is bound to this process, so the job cannot be queued. The upload was staged and is discarded; retry once a worker is running.", + "No background worker is bound to this process, or the queue did not accept the job, so nothing will run it. The staged upload is discarded (and a created job is marked failed); retry once a worker is running.", content: { "application/json": { schema: errorEnvelope } }, }, }, diff --git a/src/types/backups.ts b/src/types/backups.ts index 635b65c27..fb7b1c18d 100644 --- a/src/types/backups.ts +++ b/src/types/backups.ts @@ -58,6 +58,25 @@ export interface OffhostBackupOverview { periodHours: number; /** One row per account. Empty when off-host backup is not configured. */ rows: OffhostAccountRow[]; + /** + * Deleted or wiped accounts whose copies are still in the bucket. A count + * above zero with an error means the bucket refused the deletion (most + * often a credential without DeleteObject); the job retries nightly. + */ + pendingDeletions: { + count: number; + oldestRequestedAt: string | null; + lastFailure: string | null; + }; + /** + * Whether the bucket retires old copies by itself. Read from the bucket + * (`GetBucketLifecycleConfiguration`), cached for ten minutes; `unknown` + * when the credential may not read it. + */ + lifecycle: { + state: "configured" | "missing" | "unknown"; + expirationDays: number | null; + }; } export interface BackupsList { diff --git a/tests/integration/admin-backups-canonical-roundtrip.test.ts b/tests/integration/admin-backups-canonical-roundtrip.test.ts index 44516b544..f26ff943c 100644 --- a/tests/integration/admin-backups-canonical-roundtrip.test.ts +++ b/tests/integration/admin-backups-canonical-roundtrip.test.ts @@ -549,7 +549,11 @@ describe("canonical disaster-recovery backup round-trip", () => { const offhostObject = s3.objects.get(`2026-07-02/user-${ownerId}.json.enc`); expect(offhostObject).toBeDefined(); const payload = parseBackupPayload( - decryptBackup(offhostObject!, Buffer.from(OFFHOST_ENCRYPTION_KEY, "hex")), + decryptBackup( + offhostObject!, + Buffer.from(OFFHOST_ENCRYPTION_KEY, "hex"), + `2026-07-02/user-${ownerId}.json.enc`, + ), ); expect(payload).toMatchObject({ schemaVersion: "2", diff --git a/tests/integration/apple-health-import-worker-failures.test.ts b/tests/integration/apple-health-import-worker-failures.test.ts index b10a0ed21..2793396ad 100644 --- a/tests/integration/apple-health-import-worker-failures.test.ts +++ b/tests/integration/apple-health-import-worker-failures.test.ts @@ -219,3 +219,28 @@ describe("apple health import worker — failure paths (issue #486)", () => { expect(existsSync(uploadPath)).toBe(false); }); }); + +describe("an account deleted before the worker reaches its upload", () => { + it("removes the staged export and does not fail on the missing account", async () => { + const user = await createUser("deleted-before-import"); + const uploadPath = join( + scratchDir, + "healthlog-apple-health-import-gone.bin", + ); + writeFileSync(uploadPath, "the export, in plain text"); + // The account goes, and its ImportJob row with it by cascade. + await getPrismaClient().user.delete({ where: { id: user.id } }); + + const outcome = await handleAppleHealthImport( + bossJob("boss-gone-1", { + userId: user.id, + uploadPath, + uploadBytes: 25, + enqueuedAt: new Date().toISOString(), + }), + ); + expect(outcome).toMatchObject({ ok: true }); + expect(existsSync(uploadPath)).toBe(false); + expect(await getPrismaClient().importJob.count()).toBe(0); + }); +}); diff --git a/tests/integration/backup-key-needs.test.ts b/tests/integration/backup-key-needs.test.ts new file mode 100644 index 000000000..206c38e97 --- /dev/null +++ b/tests/integration/backup-key-needs.test.ts @@ -0,0 +1,126 @@ +/** + * A stored backup records the keys its content needs, and the encryption view + * says a retired key is still needed while any backup lists it. + * + * Both ends and the pipe: the copy is written by the real `storeBackupBlob`, + * the key ids are read from the JSON on its way into the pieces, and the + * admin status route reads them back. The case the view used to get wrong is + * the one asserted: every row rotated, the badge said the old key was safe to + * drop, and a backup still needed it. + * + * Mutations that must turn this red: drop the scanner from `storeBackupBlob` + * (nothing is recorded, the view has no key to name), or leave + * `retiredKeysStillNeeded` out of `safeToDropRetiredKeys` (the badge says + * safe again). + */ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +import { _resetCryptoCacheForTests, encrypt } from "@/lib/crypto"; + +import { cookieJar, headerJar } from "./mock-next-headers"; +import { getPrismaClient, truncateAllTables } from "./setup"; + +vi.mock("next/headers", async () => { + const { cookieJar, headerJar } = await import("./mock-next-headers"); + return { + headers: vi.fn(async () => ({ + get: (name: string) => headerJar.get(name.toLowerCase()) ?? null, + })), + cookies: vi.fn(async () => ({ + get: (name: string) => { + const value = cookieJar.get(name); + return value ? { name, value } : undefined; + }, + set: (name: string, value: string) => { + cookieJar.set(name, value); + }, + delete: (name: string) => { + cookieJar.delete(name); + }, + })), + }; +}); + +vi.mock("@/lib/db-compat", () => ({ + ensureDbCompatibility: vi.fn().mockResolvedValue(undefined), +})); + +const saved = { + key: process.env.ENCRYPTION_KEY, + keys: process.env.ENCRYPTION_KEYS, + active: process.env.ENCRYPTION_ACTIVE_KEY_ID, +}; + +function useKeys(active: string) { + process.env.ENCRYPTION_KEY = ""; + process.env.ENCRYPTION_KEYS = JSON.stringify({ + old: "11".repeat(32), + cur: "22".repeat(32), + }); + process.env.ENCRYPTION_ACTIVE_KEY_ID = active; + _resetCryptoCacheForTests(); +} + +beforeEach(async () => { + await truncateAllTables(getPrismaClient()); + cookieJar.clear(); + headerJar.clear(); +}); + +afterEach(() => { + process.env.ENCRYPTION_KEY = saved.key ?? ""; + process.env.ENCRYPTION_KEYS = saved.keys ?? ""; + process.env.ENCRYPTION_ACTIVE_KEY_ID = saved.active ?? ""; + _resetCryptoCacheForTests(); +}); + +describe("the keys stored backups still need", () => { + it("names a retired key a backup's content was written under", async () => { + const prisma = getPrismaClient(); + const admin = await prisma.user.create({ + data: { username: "keys-admin", email: "k@example.test", role: "ADMIN" }, + }); + const session = await prisma.session.create({ + data: { userId: admin.id, expiresAt: new Date(Date.now() + 60_000) }, + }); + cookieJar.set("healthlog_session", session.id); + + const { storeBackupBlob } = await import("@/lib/export/store-backup-blob"); + // A copy taken before the rotation: its note was written under "old". + useKeys("old"); + const note = Buffer.from(encrypt("before"), "utf8").toString("base64"); + // The copy itself is stored after the rotation, so its envelope is + // sealed under "cur" and only its content needs "old". + useKeys("cur"); + await storeBackupBlob( + prisma, + { userId: admin.id, type: "MANUAL_UPLOAD_KEYS" }, + async (write) => { + await write('{"userId":"x","measurements":[{"notesEncrypted":'); + await write(`"${note}"}]}`); + }, + ); + const row = await prisma.dataBackup.findFirstOrThrow({ + where: { type: "MANUAL_UPLOAD_KEYS" }, + }); + expect(row.innerKeyIds).toEqual(["old"]); + expect(row.innerKeyIdsRecorded).toBe(true); + + const { GET } = await import("@/app/api/admin/encryption/status/route"); + const res = await GET(); + expect(res.status).toBe(200); + const body = (await res.json()).data as { + rotationComplete: boolean; + safeToDropRetiredKeys: boolean; + backups: { + stored: Array<{ keyId: string; copies: number }>; + retiredKeysStillNeeded: string[]; + }; + }; + expect(body.backups.stored).toEqual([ + expect.objectContaining({ keyId: "old", copies: 1 }), + ]); + expect(body.backups.retiredKeysStillNeeded).toEqual(["old"]); + expect(body.safeToDropRetiredKeys).toBe(false); + }); +}); diff --git a/tests/integration/backup-restore-boundaries.test.ts b/tests/integration/backup-restore-boundaries.test.ts new file mode 100644 index 000000000..a715c3008 --- /dev/null +++ b/tests/integration/backup-restore-boundaries.test.ts @@ -0,0 +1,262 @@ +/** + * Two refusals a restore has to make before it changes anything. + * + * 1. The file's inner ciphertext was written under a key this host no + * longer has. The envelope opens (it is sealed under the current key), + * the schema passes, and without the check the restore would write back + * rows no reader can open. Refused with the key named; the account keeps + * what it had. + * + * 2. The file attaches the account's rows to rows another account owns: a + * custom mood tag filed under another account's category. The foreign + * key is satisfied, so only the tenant check stands in the way. Refused; + * the transaction rolls back and the account keeps what it had. + * + * Mutations that must turn these red: drop the `keyProblem` refusal in + * `restoreBackup` (case 1 restores and the note comes back unreadable), or the + * `assertNoNewForeignReferences` call (case 2 commits the foreign link). + */ +import { afterEach, beforeEach, describe, expect, it } from "vitest"; + +import { _resetCryptoCacheForTests, encrypt } from "@/lib/crypto"; +import { STORED_BACKUP_SELECT } from "@/lib/export/stored-backup"; +import { restoreBackup } from "@/lib/export/restore-backup"; + +import { getPrismaClient, truncateAllTables } from "./setup"; + +const saved = { + key: process.env.ENCRYPTION_KEY, + keys: process.env.ENCRYPTION_KEYS, + active: process.env.ENCRYPTION_ACTIVE_KEY_ID, +}; + +function useKeys(keys: Record, active: string) { + process.env.ENCRYPTION_KEY = ""; + process.env.ENCRYPTION_KEYS = JSON.stringify(keys); + process.env.ENCRYPTION_ACTIVE_KEY_ID = active; + _resetCryptoCacheForTests(); +} + +beforeEach(async () => { + await truncateAllTables(getPrismaClient()); +}); + +afterEach(() => { + process.env.ENCRYPTION_KEY = saved.key ?? ""; + process.env.ENCRYPTION_KEYS = saved.keys ?? ""; + process.env.ENCRYPTION_ACTIVE_KEY_ID = saved.active ?? ""; + _resetCryptoCacheForTests(); +}); + +async function seedOwner(username: string) { + return getPrismaClient().user.create({ + data: { username, email: `${username}@example.test` }, + }); +} + +async function storeBackup(userId: string, payload: unknown) { + const prisma = getPrismaClient(); + const row = await prisma.dataBackup.create({ + data: { + userId, + type: "MANUAL_UPLOAD_BOUNDARY_TEST", + data: encrypt(JSON.stringify(payload)), + }, + }); + return prisma.dataBackup.findUniqueOrThrow({ + where: { id: row.id }, + select: STORED_BACKUP_SELECT, + }); +} + +function basePayload(userId: string) { + return { + schemaVersion: "1", + exportedAt: "2026-05-09T10:00:00.000Z", + userId, + measurements: [] as unknown[], + medications: [], + intakeEvents: [], + moodEntries: [], + }; +} + +describe("restore refuses before it changes anything", () => { + it("refuses a file whose inner ciphertext needs a key the host dropped", async () => { + const prisma = getPrismaClient(); + useKeys({ old: "11".repeat(32), cur: "22".repeat(32) }, "old"); + const owner = await seedOwner("key-owner"); + const note = Buffer.from(encrypt("written before the rotation"), "utf8"); + const payload = basePayload(owner.id); + payload.measurements = [ + { + type: "WEIGHT", + value: 80, + unit: "kg", + measuredAt: "2026-05-08T07:00:00.000Z", + source: "MANUAL", + notesEncrypted: note.toString("base64"), + }, + ]; + + // The rotation ran and the operator dropped the old key; the stored copy's + // envelope is sealed under the key that remains. + useKeys({ cur: "22".repeat(32) }, "cur"); + const backup = await storeBackup(owner.id, payload); + const kept = await prisma.measurement.create({ + data: { + userId: owner.id, + type: "WEIGHT", + value: 99, + unit: "kg", + measuredAt: new Date("2026-05-01T07:00:00.000Z"), + source: "MANUAL", + }, + }); + + const outcome = await restoreBackup({ + backup, + actorUserId: owner.id, + ipAddress: null, + restoreInstanceSettings: false, + }); + expect(outcome.ok).toBe(false); + if (outcome.ok) return; + expect(outcome.code).toBe("backup.key.missing"); + expect(outcome.message).toContain("'old'"); + expect(outcome.meta?.keyIds).toEqual(["old"]); + + const rows = await prisma.measurement.findMany({ + where: { userId: owner.id }, + }); + expect(rows.map((r) => r.id)).toEqual([kept.id]); + + // With the key back, the same copy restores and the note opens. + useKeys({ old: "11".repeat(32), cur: "22".repeat(32) }, "cur"); + const again = await restoreBackup({ + backup, + actorUserId: owner.id, + ipAddress: null, + restoreInstanceSettings: false, + }); + expect(again.ok).toBe(true); + const restored = await prisma.measurement.findFirstOrThrow({ + where: { userId: owner.id }, + }); + expect(restored.notesEncrypted).not.toBeNull(); + expect( + Buffer.from(restored.notesEncrypted!).toString("utf8").startsWith("old."), + ).toBe(true); + }); + + it("refuses a file that files a tag under another account's category", async () => { + const prisma = getPrismaClient(); + const owner = await seedOwner("tenant-owner"); + const other = await seedOwner("tenant-other"); + const foreign = await prisma.moodTagCategory.create({ + data: { + userId: other.id, + key: "custom:other-category", + labelKey: "mood.tagCategory.custom", + }, + }); + const kept = await prisma.moodEntry.create({ + data: { + userId: owner.id, + date: "2026-05-01", + mood: "OKAY", + score: 3, + moodLoggedAt: new Date("2026-05-01T20:00:00.000Z"), + }, + }); + + const payload = { + ...basePayload(owner.id), + moodEntries: [ + { + date: "2026-05-08", + mood: "GUT", + score: 4, + tags: null, + source: "MOODLOG", + loggedAt: "2026-05-08T20:00:00.000Z", + structuredTags: ["custom:owner-tag"], + }, + ], + customMoodTags: [ + { + key: "custom:owner-tag", + labelKey: "mood.tag.custom", + categoryId: foreign.id, + kind: "BINARY", + }, + ], + }; + const backup = await storeBackup(owner.id, payload); + + const outcome = await restoreBackup({ + backup, + actorUserId: owner.id, + ipAddress: null, + restoreInstanceSettings: false, + }); + expect(outcome.ok).toBe(false); + if (outcome.ok) return; + expect(outcome.code).toBe("backup.foreign_reference"); + + expect( + await prisma.moodTag.count({ where: { key: "custom:owner-tag" } }), + ).toBe(0); + const moods = await prisma.moodEntry.findMany({ + where: { userId: owner.id }, + }); + expect(moods.map((m) => m.id)).toEqual([kept.id]); + expect(outcome.message).toContain("mood_tags.category_id"); + }); + + it("restores a file whose tag uses the account's own category", async () => { + const prisma = getPrismaClient(); + const owner = await seedOwner("tenant-own"); + const payload = { + ...basePayload(owner.id), + moodEntries: [ + { + date: "2026-05-08", + mood: "GUT", + score: 4, + tags: null, + source: "MOODLOG", + loggedAt: "2026-05-08T20:00:00.000Z", + structuredTags: ["custom:own-tag"], + }, + ], + customMoodTagCategories: [ + { + id: "own-category-id", + key: "custom:own-category", + labelKey: "mood.tagCategory.custom", + }, + ], + customMoodTags: [ + { + key: "custom:own-tag", + labelKey: "mood.tag.custom", + categoryId: "own-category-id", + kind: "BINARY", + }, + ], + }; + const backup = await storeBackup(owner.id, payload); + const outcome = await restoreBackup({ + backup, + actorUserId: owner.id, + ipAddress: null, + restoreInstanceSettings: false, + }); + expect(outcome.ok).toBe(true); + const tag = await prisma.moodTag.findUniqueOrThrow({ + where: { key: "custom:own-tag" }, + }); + expect(tag.userId).toBe(owner.id); + }); +}); diff --git a/tests/integration/offhost-backup-streaming-memory.test.ts b/tests/integration/offhost-backup-streaming-memory.test.ts index c5193fd65..0c253dd69 100644 --- a/tests/integration/offhost-backup-streaming-memory.test.ts +++ b/tests/integration/offhost-backup-streaming-memory.test.ts @@ -273,7 +273,9 @@ describe("off-host backup under a memory budget", () => { ).toBeLessThan(STREAM_BUDGET_BYTES); // An object that writes but does not read is worse than none. - const restored = JSON.parse(decryptBackup(s3.kept!, BACKUP_KEY)) as { + const restored = JSON.parse( + decryptBackup(s3.kept!, BACKUP_KEY, objectKey), + ) as { measurements: Array<{ deletedAt: string | null }>; moodEntries: unknown[]; intakeEvents: unknown[]; diff --git a/tests/integration/offhost-purge-on-deletion.test.ts b/tests/integration/offhost-purge-on-deletion.test.ts new file mode 100644 index 000000000..9c96deb75 --- /dev/null +++ b/tests/integration/offhost-purge-on-deletion.test.ts @@ -0,0 +1,157 @@ +/** + * Deleting an account, or wiping its data, takes its off-host copies out of + * the bucket. + * + * The bucket is replaced at the client boundary (`getS3Client`) by an + * in-memory map; everything above it is real: the route, its transaction, the + * request row it commits and the purge job that works it. Mutations that must + * turn this red: drop `requestOffhostPurge` from either route (no request is + * written and the objects stay), or write the request outside the deletion's + * transaction and let it fail (the deletion commits without one). + */ +import { NextRequest } from "next/server"; +import { beforeEach, describe, expect, it, vi } from "vitest"; + +import { cookieJar, headerJar } from "./mock-next-headers"; +import { getPrismaClient, truncateAllTables } from "./setup"; + +const bucket = vi.hoisted(() => new Set()); + +vi.mock("@/lib/jobs/offhost-backup", async (importOriginal) => { + const actual = + await importOriginal(); + return { + ...actual, + getS3Client: vi.fn(async () => ({ + putObject: vi.fn(), + putStream: vi.fn(), + getObject: vi.fn(), + headObject: vi.fn(), + listObjects: vi.fn(async (prefix: string) => + [...bucket].filter((k) => k.startsWith(prefix)).map((key) => ({ key })), + ), + deleteObject: vi.fn(async (key: string) => { + bucket.delete(key); + }), + })), + }; +}); + +vi.mock("next/headers", async () => { + const { cookieJar, headerJar } = await import("./mock-next-headers"); + return { + headers: vi.fn(async () => ({ + get: (name: string) => headerJar.get(name.toLowerCase()) ?? null, + })), + cookies: vi.fn(async () => ({ + get: (name: string) => { + const value = cookieJar.get(name); + return value ? { name, value } : undefined; + }, + set: (name: string, value: string) => { + cookieJar.set(name, value); + }, + delete: (name: string) => { + cookieJar.delete(name); + }, + })), + }; +}); + +vi.mock("@/lib/db-compat", () => ({ + ensureDbCompatibility: vi.fn().mockResolvedValue(undefined), +})); + +beforeEach(async () => { + await truncateAllTables(getPrismaClient()); + cookieJar.clear(); + headerJar.clear(); + bucket.clear(); + process.env.BACKUP_S3_ENDPOINT = "https://r2.example"; + process.env.BACKUP_S3_BUCKET = "hl-backups"; + process.env.BACKUP_S3_ACCESS_KEY = "AKIA"; + process.env.BACKUP_S3_SECRET_KEY = "secret"; + process.env.BACKUP_ENCRYPTION_KEY = "ab".repeat(32); +}); + +async function seedUserWithSession(username: string) { + const prisma = getPrismaClient(); + const user = await prisma.user.create({ + data: { username, email: `${username}@example.test` }, + }); + const session = await prisma.session.create({ + data: { userId: user.id, expiresAt: new Date(Date.now() + 3_600_000) }, + }); + cookieJar.set("healthlog_session", session.id); + return user.id; +} + +function seedBucket(userId: string, otherId: string) { + for (const day of ["2026-09-01", "2026-09-02", "2026-09-03"]) { + bucket.add(`${day}/user-${userId}.json.enc`); + bucket.add(`${day}/user-${otherId}.json.enc`); + } + bucket.add("_healthcheck/1.bin"); +} + +describe("off-host copies leave with the account", () => { + it("account deletion removes every copy of that account", async () => { + const prisma = getPrismaClient(); + const other = await prisma.user.create({ + data: { username: "neighbour", email: "neighbour@example.test" }, + }); + const userId = await seedUserWithSession("leaving"); + seedBucket(userId, other.id); + + const { DELETE } = await import("@/app/api/settings/account/route"); + const res = await DELETE( + new NextRequest("http://localhost/api/settings/account", { + method: "DELETE", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ confirm: "DELETE_ACCOUNT" }), + }), + ); + expect(res.status).toBe(200); + + const pending = await prisma.offhostPurgeRequest.findMany(); + expect(pending.map((r) => [r.subjectId, r.reason])).toEqual([ + [userId, "account_deleted"], + ]); + + const { processOffhostPurges } = await import("@/lib/jobs/offhost-purge"); + const report = await processOffhostPurges(prisma); + expect(report).toMatchObject({ completed: 1, objectsDeleted: 3 }); + expect([...bucket].filter((k) => k.includes(userId))).toEqual([]); + expect([...bucket].filter((k) => k.includes(other.id))).toHaveLength(3); + expect(bucket.has("_healthcheck/1.bin")).toBe(true); + expect(await prisma.offhostPurgeRequest.count()).toBe(0); + }); + + it("delete-all-data removes every copy and leaves the receipt", async () => { + const prisma = getPrismaClient(); + const other = await prisma.user.create({ + data: { username: "neighbour2", email: "neighbour2@example.test" }, + }); + const userId = await seedUserWithSession("wiping"); + seedBucket(userId, other.id); + + const { DELETE } = await import("@/app/api/settings/data/route"); + const res = await DELETE( + new NextRequest("http://localhost/api/settings/data", { + method: "DELETE", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ confirm: "DELETE" }), + }), + ); + expect(res.status).toBe(200); + + const { processOffhostPurges } = await import("@/lib/jobs/offhost-purge"); + await processOffhostPurges(prisma); + expect([...bucket].filter((k) => k.includes(userId))).toEqual([]); + expect([...bucket].filter((k) => k.includes(other.id))).toHaveLength(3); + const receipt = await prisma.auditLog.findFirst({ + where: { action: "offhost.backup.purged", userId }, + }); + expect(receipt).not.toBeNull(); + }); +});