Merge branch 'fix/backup-rotation-and-deletion' into release/v1.39.3

# Conflicts: # docs/api/openapi.yaml # src/lib/jobs/apple-health-import-worker.ts
2026-11-12 06:06:10 +00:00 · 2026-09-26 21:48:20 +02:00
commit 4409529c89
@@ -13353,6 +13353,14 @@ paths:
              schema:
                $ref: "#/components/schemas/ErrorEnvelope"
        "401": *a1
+        "409":
+          description: An import for this account is already queued or running, and these bytes are not the ones it carries (the
+            same bytes resolve to that job as a 202). `meta.errorCode` is `import.apple_health.busy` and `meta.jobId`
+            names the running job; poll it, then upload again. The upload was discarded.
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/ErrorEnvelope"
        "413":
          description: The declared `Content-Length` exceeds 1.5 GB. A body that exceeds the cap without declaring it is refused
            by the streaming sink instead, which surfaces as the 422 below.
@@ -13374,8 +13382,8 @@ paths:
              schema:
                $ref: "#/components/schemas/ErrorEnvelope"
        "503":
-          description: No background worker is bound to this process, so the job cannot be queued. The upload was staged and is
-            discarded; retry once a worker is running.
+          description: No background worker is bound to this process, or the queue did not accept the job, so nothing will run it.
+            The staged upload is discarded (and a created job is marked failed); retry once a worker is running.
          content:
            application/json:
              schema:
@@ -31389,31 +31397,31 @@ components:
                anamnesis.fact.invalidValue; auth — auth.admin.required, auth.already_authenticated,
                auth.mfa.code_invalid, auth.missing, auth.refresh.invalid, auth.refresh.reuse, auth.refresh.revoked,
                auth.reproof.failed, auth.reproof.required, auth.reproof.too_weak, auth.scope.insufficient,
-                auth.stepup.required, auth.token.expired, auth.token.invalid; backup — backup.payload.undecryptable,
-                backup.restore.active, backup.section.missing, backup.upload.too_large; body-site — body-site.invalid;
-                coach — coach.budget.exceeded, coach.conversation.invalidTitle, coach.conversation.notFound,
-                coach.fenced.attachmentConflict, coach.fenced.attachmentLimit, coach.fenced.attachmentNotFound,
-                coach.fenced.attachmentNotIndexed, coach.fenced.attachmentRequired, coach.fenced.attachmentUnavailable,
-                coach.fenced.invalid, coach.request.invalid; coach-prefs — coach-prefs.body.invalid_json,
-                coach-prefs.body.invalid_shape; consent — consent.ai.required; cycle — cycle.bulk.invalid,
-                cycle.bulk.too_large, cycle.calendar.invalid, cycle.calendar.range, cycle.cycles.invalid,
-                cycle.day-log.conflict, cycle.day-log.invalid, cycle.disabled, cycle.period.invalid,
-                cycle.symptom.custom.invalid, cycle.symptom.custom.limit; cycle-prefs — cycle-prefs.body.invalid_json,
-                cycle-prefs.invalid; disable-coach — disable-coach.body.invalid_json; documents —
-                documents.chat.conversationNotFound, documents.chat.invalid, documents.inbound.alreadyConfirmed,
-                documents.inbound.alreadyPartlyConfirmed, documents.inbound.budgetExceeded,
-                documents.inbound.decryptFailed, documents.inbound.encounterNotFound, documents.inbound.episodeNotFound,
-                documents.inbound.extractFailed, documents.inbound.factNotFound, documents.inbound.factNotPending,
-                documents.inbound.factTypeMismatch, documents.inbound.fileTooLarge, documents.inbound.fileType,
-                documents.inbound.invalidMetadata, documents.inbound.invalidQuery, documents.inbound.invalidUpdate,
-                documents.inbound.localOcrDisabled, documents.inbound.notFound, documents.inbound.notIndexed,
-                documents.inbound.pdfNeedsAnthropic, documents.inbound.providerUnsupported,
-                documents.inbound.quotaExceeded, documents.inbound.rateLimited, documents.inbound.restoreDuplicate,
-                documents.inbound.restoreGone, documents.inbound.sourceAliasLimit, documents.inbound.uploadBusy,
-                documents.inbound.uploadTimeout, documents.inbound.vaccinationNotFound, documents.sources.authRefused,
-                documents.sources.badResponse, documents.sources.browserOnly, documents.sources.invalidAddress,
-                documents.sources.linkTargetNotFound, documents.sources.notConnected, documents.sources.notFound,
-                documents.sources.organizationRequired, documents.sources.originNotAllowed,
+                auth.stepup.required, auth.token.expired, auth.token.invalid; backup — backup.foreign_reference,
+                backup.key.missing, backup.payload.undecryptable, backup.restore.active, backup.section.missing,
+                backup.upload.too_large; body-site — body-site.invalid; coach — coach.budget.exceeded,
+                coach.conversation.invalidTitle, coach.conversation.notFound, coach.fenced.attachmentConflict,
+                coach.fenced.attachmentLimit, coach.fenced.attachmentNotFound, coach.fenced.attachmentNotIndexed,
+                coach.fenced.attachmentRequired, coach.fenced.attachmentUnavailable, coach.fenced.invalid,
+                coach.request.invalid; coach-prefs — coach-prefs.body.invalid_json, coach-prefs.body.invalid_shape;
+                consent — consent.ai.required; cycle — cycle.bulk.invalid, cycle.bulk.too_large, cycle.calendar.invalid,
+                cycle.calendar.range, cycle.cycles.invalid, cycle.day-log.conflict, cycle.day-log.invalid,
+                cycle.disabled, cycle.period.invalid, cycle.symptom.custom.invalid, cycle.symptom.custom.limit;
+                cycle-prefs — cycle-prefs.body.invalid_json, cycle-prefs.invalid; disable-coach —
+                disable-coach.body.invalid_json; documents — documents.chat.conversationNotFound,
+                documents.chat.invalid, documents.inbound.alreadyConfirmed, documents.inbound.alreadyPartlyConfirmed,
+                documents.inbound.budgetExceeded, documents.inbound.decryptFailed, documents.inbound.encounterNotFound,
+                documents.inbound.episodeNotFound, documents.inbound.extractFailed, documents.inbound.factNotFound,
+                documents.inbound.factNotPending, documents.inbound.factTypeMismatch, documents.inbound.fileTooLarge,
+                documents.inbound.fileType, documents.inbound.invalidMetadata, documents.inbound.invalidQuery,
+                documents.inbound.invalidUpdate, documents.inbound.localOcrDisabled, documents.inbound.notFound,
+                documents.inbound.notIndexed, documents.inbound.pdfNeedsAnthropic,
+                documents.inbound.providerUnsupported, documents.inbound.quotaExceeded, documents.inbound.rateLimited,
+                documents.inbound.restoreDuplicate, documents.inbound.restoreGone, documents.inbound.sourceAliasLimit,
+                documents.inbound.uploadBusy, documents.inbound.uploadTimeout, documents.inbound.vaccinationNotFound,
+                documents.sources.authRefused, documents.sources.badResponse, documents.sources.browserOnly,
+                documents.sources.invalidAddress, documents.sources.linkTargetNotFound, documents.sources.notConnected,
+                documents.sources.notFound, documents.sources.organizationRequired, documents.sources.originNotAllowed,
                documents.sources.permissionMissing, documents.sources.rateLimited, documents.sources.redirected,
                documents.sources.tokenRequired, documents.sources.unavailable, documents.sources.unreachable,
                documents.sources.versionTooOld; encounter — encounter.invalid, encounter.link.invalid,
@@ -31423,13 +31431,13 @@ components:
                feedback — feedback.body.invalid; health_score_config — health_score_config.invalid,
                health_score_config.too_narrow; illness — illness.day-log.invalid, illness.disabled,
                illness.episode.chronic-no-resolve, illness.episode.invalid, illness.episode.invalid-window,
-                illness.episode.parent-not-found, illness.insights.invalid; import — import.write_failed; ingest —
-                ingest.rate_limited; insights — insights.generate.budgetExceeded, insights.generate.outboundScreened,
-                insights.generate.profileScopeChanged, insights.generate.scopeChanged; labs —
-                labs.biomarker.create.invalid, labs.biomarker.duplicate, labs.biomarker.notFound,
-                labs.biomarker.update.invalid, labs.biomarker.update.referenceRangeInvalid, labs.create.invalid,
-                labs.ocr.budgetExceeded, labs.ocr.extractFailed, labs.ocr.fileTooLarge, labs.ocr.fileType,
-                labs.ocr.localOcrDisabled, labs.ocr.pdfNeedsAnthropic, labs.ocr.providerUnsupported,
+                illness.episode.parent-not-found, illness.insights.invalid; import — import.apple_health.busy,
+                import.write_failed; ingest — ingest.rate_limited; insights — insights.generate.budgetExceeded,
+                insights.generate.outboundScreened, insights.generate.profileScopeChanged,
+                insights.generate.scopeChanged; labs — labs.biomarker.create.invalid, labs.biomarker.duplicate,
+                labs.biomarker.notFound, labs.biomarker.update.invalid, labs.biomarker.update.referenceRangeInvalid,
+                labs.create.invalid, labs.ocr.budgetExceeded, labs.ocr.extractFailed, labs.ocr.fileTooLarge,
+                labs.ocr.fileType, labs.ocr.localOcrDisabled, labs.ocr.pdfNeedsAnthropic, labs.ocr.providerUnsupported,
                labs.ocr.rateLimited, labs.restore.invalid, labs.result.notFound, labs.update.invalid,
                labs.update.linkedFieldsImmutable, labs.update.numericExpected, labs.update.qualitativeExpected,
                labs.update.referenceRangeInvalid, labs.update.sourceReferenceRangeInvalid; managed_profile —
@@ -73040,21 +73048,26 @@ components:
          type: string
          description: "Stable reason: `backup_not_found`, `backup_changed`, `backup.payload.undecryptable`, `schema_invalid`,
            `incompatible_schema_version`, `owner_mismatch`, `owner_not_found`, `backup.section.missing`,
-            `document_ciphertext_missing`, `time_budget`, `transaction_failed`, `interrupted`, `not_started`,
-            `enqueue_failed`, `failed_after_commit`, `unexpected`. Every code but `failed_after_commit` means the
-            account was not changed."
+            `document_ciphertext_missing`, `backup.key.missing`, `backup.foreign_reference`, `time_budget`,
+            `transaction_failed`, `interrupted`, `not_started`, `enqueue_failed`, `failed_after_commit`, `unexpected`.
+            Every code but `failed_after_commit` means the account was not changed."
        message:
          type: string
        sections:
          type: array
          items:
            type: string
+        keyIds:
+          type: array
+          items:
+            type: string
      required:
        - code
        - message
      additionalProperties: false
      description: Why the restore did not happen, in a sentence an operator can act on. `sections` names the missing sections
-        for `backup.section.missing`.
+        for `backup.section.missing`; `keyIds` names the encryption key ids the file needs and this server cannot open
+        it with, for `backup.key.missing`.
    AdminBackupRestoreJobResponse:
      type: object
      properties:
@@ -16,13 +16,23 @@ container (queue `data-backup-offhost`). Object key layout:

 ```
 magic   = "HLBK"           (4 bytes, ASCII)
-version = 0x03             (1 byte)
+version = 0x04             (1 byte)
+keyIdLen, keyId            (1 byte + 12 ASCII hex: the first 12 hex digits of SHA-256(key))
 iv      = 12 random bytes  (AES-GCM nonce)
-ciphertext = N bytes       (AES-256-GCM over gzip(JSON dump), key = BACKUP_ENCRYPTION_KEY)
+ciphertext = N bytes       (AES-256-GCM over gzip(JSON dump), key = BACKUP_ENCRYPTION_KEY,
+                            associated data = "healthlog/offhost-backup/v4|<object key>")
 authTag = 16 bytes         (AES-GCM tag, trailing)
 ```

-Three versions exist in the wild and all three restore. `0x01` encrypted the
+Version `0x04` (from v1.39.3) adds two things. The key id says which off-host
+key an object needs, so a reader holding a retired key as well finds the right
+one. The associated data is the object's own key in the bucket,
+`YYYY-MM-DD/user-<id>.json.enc`: an object copied to another account's name or
+another date does not decrypt, where before it opened as if it belonged there.
+The restore script passes the key it downloaded, so nothing changes for an
+operator.
+
+Four versions exist in the wild and all four restore. `0x01` encrypted the
 JSON directly and `0x02` gzipped it first; both carry the tag in FRONT of the
 ciphertext, which is what made them impossible to write a piece at a time —
 GCM only produces the tag once the last block is in, so a leading tag means the
@@ -33,32 +43,59 @@ the restore script needs no flag to tell them apart.

 ## Required env vars

-| Var                     | Required | Notes                                                                |
-| ----------------------- | -------- | -------------------------------------------------------------------- |
-| `BACKUP_ENCRYPTION_KEY` | yes      | 64 hex chars or 32-byte base64. **Different from `ENCRYPTION_KEY`.** |
-| `BACKUP_S3_ENDPOINT`    | yes      | e.g. `https://<account>.r2.cloudflarestorage.com`                    |
-| `BACKUP_S3_BUCKET`      | yes      |                                                                      |
-| `BACKUP_S3_ACCESS_KEY`  | yes      |                                                                      |
-| `BACKUP_S3_SECRET_KEY`  | yes      |                                                                      |
-| `BACKUP_S3_REGION`      | no       | defaults to `auto` (Cloudflare R2)                                   |
-| `BACKUP_RETENTION_DAYS` | no       | not read by the app; the number for your bucket's lifecycle rule     |
+| Var                               | Required | Notes                                                                               |
+| --------------------------------- | -------- | ----------------------------------------------------------------------------------- |
+| `BACKUP_ENCRYPTION_KEY`           | yes      | 64 hex chars or 32-byte base64. **Different from `ENCRYPTION_KEY`.**                |
+| `BACKUP_S3_ENDPOINT`              | yes      | e.g. `https://<account>.r2.cloudflarestorage.com`                                   |
+| `BACKUP_S3_BUCKET`                | yes      |                                                                                     |
+| `BACKUP_S3_ACCESS_KEY`            | yes      |                                                                                     |
+| `BACKUP_S3_SECRET_KEY`            | yes      |                                                                                     |
+| `BACKUP_S3_REGION`                | no       | defaults to `auto` (Cloudflare R2)                                                  |
+| `BACKUP_RETENTION_DAYS`           | no       | not read by the app; the number for your bucket's lifecycle rule                    |
+| `BACKUP_ENCRYPTION_PREVIOUS_KEYS` | no       | retired off-host keys, comma-separated, read only (see "Rotating the off-host key") |

 ## Bucket permissions

-The worker needs `PutObject`, `GetObject` and `AbortMultipartUpload`. It never
-calls `DeleteObject` on a backup key, so a compromised worker cannot wipe the
-history; `AbortMultipartUpload` only reaches an upload that same worker started
-and is what clears the parts of a run that failed halfway. Without it, a failed
-upload leaves parts that are billed and do not show in a bucket listing. On
-Cloudflare R2 the **Object Read & Write** token already covers all three.
+The worker needs `PutObject`, `GetObject`, `ListBucket`, `DeleteObject` and
+`AbortMultipartUpload`. On Cloudflare R2 the **Object Read & Write** token
+covers all of them.

-## Bucket lifecycle (recommended)
+`DeleteObject` is needed because the app deletes objects in two cases, and
+only those:

-The worker never deletes a backup object: its grant covers PutObject,
-GetObject and AbortMultipartUpload only, so a compromised worker cannot
-wipe the history, and nothing in the app reads `BACKUP_RETENTION_DAYS`.
-Expiry is the storage provider's lifecycle rule, set to match the number
-you wrote there:
+- **An account is deleted, or its data is wiped.** Deleting an account,
+  deleting a managed profile and "Delete all data" each write a deletion
+  request in the same transaction as the deletion. The `offhost-backup-purge` job lists the bucket, deletes every
+  `YYYY-MM-DD/user-<id>.json.enc` of that account and removes the request once
+  nothing of the account is left. It runs right after the deletion and again
+  every night at 03:40. A copy the nightly upload was writing while the account
+  was deleted is removed by the upload itself.
+- **The connection test** deletes its own `_healthcheck/<ts>.bin`.
+
+The admin wipe of every account deletes nothing in the bucket. It is you
+clearing your own host, the bucket is yours, and its copies are the way back
+from a wipe pressed by mistake. If you are handing the host on, empty the
+bucket yourself.
+
+Earlier versions of this page said the worker never deletes an object. That
+was the design until v1.39.3, and it meant a deleted account's copies stayed
+in the bucket for as long as the lifecycle rule kept them, or forever without
+one. If the bucket refuses the delete (a credential without `DeleteObject`),
+the request stays, the job retries it every night, and **Admin → Backups →
+Off-host** shows how many deletions are outstanding and the bucket's answer.
+
+`AbortMultipartUpload` only reaches an upload the same worker started and is
+what clears the parts of a run that failed halfway. Without it, a failed
+upload leaves parts that are billed and do not show in a bucket listing.
+
+## Bucket lifecycle (required for retention)
+
+The app does not expire objects by age, and nothing in the app reads
+`BACKUP_RETENTION_DAYS`. Expiry is the storage provider's lifecycle rule, set
+to match the number you wrote there. Without one, every nightly copy of every
+account is kept forever. **Admin → Backups → Off-host** reads the bucket's
+lifecycle configuration and says so when there is none, or when the credential
+may not read it (grant `GetLifecycleConfiguration` to let it check):

 ```
 Filter: "" (all objects)
@@ -343,6 +380,28 @@ in seconds having backed up nothing is that defect and not your record.

 ### Key rotation

+Rotation reaches a stored backup's envelope and nothing inside it. The
+content of a disaster-recovery backup is the database's ciphertext, carried
+as it is stored, and a restore writes it back verbatim: a note, a document or
+a coach message inside a copy taken before a rotation still needs the key the
+rotation retired. The same holds for every copy in the off-host bucket and for
+every file an operator has downloaded. So the rule is: **keep a retired
+`ENCRYPTION_KEYS` entry until no backup you may still restore needs it.**
+**Admin → Encryption → Keys the backups still need** lists, per key id, the
+stored copies that need it with the oldest date, and when the last off-host
+copy needing it expires under the bucket's lifecycle rule. Copies written
+before v1.39.3 did not record their keys and are listed as such.
+
+Every restore, the restore preview, the admin upload and the monthly drill
+check this before anything is changed: a file whose content needs a key this
+server does not have (or an id whose key material differs) is refused with the
+key id named, and the account is left as it was.
+
+Re-encrypting the content of stored backups during rotation was considered and
+not done: it would mean decrypting, parsing, re-encrypting and re-writing every
+copy in full, and it still could not reach the off-host copies or downloaded
+files, so the rule above would be needed regardless.
+
 `scripts/rotate-encryption-key.ts` and the rotation in the admin console
 re-seal every piece under the active key, in small batches, without reading
 the backup inside. The link between a piece, its copy and its position is
@@ -367,6 +426,17 @@ read back. Restoring Monday's copy on Wednesday therefore costs the account
 everything it recorded on Tuesday. If the current state is worth keeping, take
 a fresh snapshot from the backups console before restoring the old one.

+## Rotating the off-host key
+
+`BACKUP_ENCRYPTION_KEY` is the key new objects are written under. To rotate
+it, set the new key there and move the old one into
+`BACKUP_ENCRYPTION_PREVIOUS_KEYS` (comma-separated if there are several), then
+restart. New objects are written under the new key; older objects stay
+readable because every version-4 object names its key and older objects are
+tried against each key in turn. Remove the old key once the lifecycle rule has
+retired every object written under it, that is `BACKUP_RETENTION_DAYS` after
+the restart.
+
 ## Monthly restore drill (automatic)

 Since v1.16.4 a pg-boss job (`data-restore-drill`, cron `11 4 1 * *` —
@@ -386,8 +456,13 @@ Outcomes:
 - **Failure** — empty bucket, fetch error, decryption failure (wrong or
  rotated key), malformed JSON: pages the same way. A decryption
  failure right after a `BACKUP_ENCRYPTION_KEY` change means the new
-  key cannot read the existing objects — re-encrypt or accept that
-  pre-rotation backups are only readable with the retired key.
+  key cannot read the existing objects: put the old key into
+  `BACKUP_ENCRYPTION_PREVIOUS_KEYS` (see "Rotating the off-host key").
+- **Content under a missing application key** — the object opens, but its
+  content was written under an `ENCRYPTION_KEYS` entry this server no longer
+  has, or under an id whose key material changed. The drill decrypts one value
+  per key id it finds, and pages naming the key. A restore of that object
+  would write back rows nobody can open; put the key back before restoring.
 - **Not configured** — deployments without the `BACKUP_S3_*` vars skip
  silently (wide-event warning only).

@@ -118,14 +118,34 @@ under `v1` (the synthetic id assigned to the existing `ENCRYPTION_KEY`).
   Restart. The legacy single-key fallback is now disconnected; only `v2`
   exists.

-> **Backups are covered, and were not always.** `DataBackup.data` holds every
-> weekly disaster-recovery snapshot and every uploaded pack, encrypted like
-> everything else but under a column called `data`. It was outside the
-> registry until v1.38.6, so a rotation before that release reported zero
-> rows remaining without ever reading a backup. If you rotated on an older
-> release and dropped the previous key, the stored backups are encrypted
-> under the key you removed: put that key back into `ENCRYPTION_KEYS` and
-> re-run the rotation on this release before removing it again.
+> **The rows in the database are covered. The content of a backup is not.**
+> Rotation re-encrypts every registered column, including `DataBackup.data`
+> and the pieces in `DataBackupChunk`, so every stored copy opens under the
+> new key. What it cannot change is what a disaster-recovery backup carries
+> inside: the database's ciphertext as it was stored when the copy was taken,
+> which a restore writes back verbatim. A copy taken before the rotation still
+> needs the old key for its notes, documents and coach history, and so does
+> every copy in the off-host bucket and every backup file you downloaded.
+>
+> So before step 4, also check **Admin → Encryption → Keys the backups still
+> need**. It lists, per key id, the stored copies that need it with the oldest
+> date, and when the last off-host copy needing it expires under the bucket's
+> lifecycle rule. Keep the old key in `ENCRYPTION_KEYS` until it is no longer
+> listed there (the weekly copy replaces itself within a week; delete old
+> uploaded copies you no longer want, and wait out the off-host retention), or
+> accept that those copies cannot be restored. Copies written before v1.39.3
+> did not record their keys and are listed as such: treat them as needing
+> every key that existed when they were written.
+>
+> Dropping the key early no longer fails silently: a restore, a restore
+> preview or an upload of a copy that needs a missing key is refused with the
+> key id named, and nothing is changed. Putting the key back makes the copy
+> restorable again.
+>
+> If you rotated on a release before v1.38.6 and dropped the previous key,
+> the stored backups themselves are encrypted under the key you removed: put
+> that key back into `ENCRYPTION_KEYS` and re-run the rotation on this release
+> before removing it again.

 ## Adding a third key (v2 → v3)