mirror of
https://github.com/MBombeck/HealthLog.git
synced 2026-11-10 05:12:14 +00:00
fix(backup): leave the admin wipe's off-host copies in place, and correct the backup and rotation runbooks
The admin wipe of every account is the operator clearing their own host, and the bucket's copies are the documented way back from a wipe pressed by mistake, so it writes no off-host deletion request. Deleting an account, deleting a managed profile and "Delete all data" still do. The backup runbook no longer says the worker never deletes an object: it lists the grant the worker needs (DeleteObject included), when objects are deleted, the version-4 envelope, the lifecycle check and how to rotate the off-host key. The rotation runbook no longer says backups are covered: rotation reaches a backup's envelope, not the content a restore writes back, and a retired key stays needed until no backup lists it. The privacy notice beside account deletion says the same about off-host copies. A failed import records its failure in a plain try block, and the extraction test that inflates past the 64 MiB floor gets the time it needs on a loaded runner.
This commit is contained in:
1 parent
849748c4cd
commit
7da778760e
19 files changed
+223
-153
No files matched your search
+50
-36
@@ -13201,6 +13201,14 @@ paths:
|
||||
schema:
|
||||
$ref: "#/components/schemas/ErrorEnvelope"
|
||||
"401": *a1
|
||||
"409":
|
||||
description: An import for this account is already queued or running, and these bytes are not the ones it carries (the
|
||||
same bytes resolve to that job as a 202). `meta.errorCode` is `import.apple_health.busy` and `meta.jobId`
|
||||
names the running job; poll it, then upload again. The upload was discarded.
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
$ref: "#/components/schemas/ErrorEnvelope"
|
||||
"413":
|
||||
description: The declared `Content-Length` exceeds 1.5 GB. A body that exceeds the cap without declaring it is refused
|
||||
by the streaming sink instead, which surfaces as the 422 below.
|
||||
@@ -13222,8 +13230,8 @@ paths:
|
||||
schema:
|
||||
$ref: "#/components/schemas/ErrorEnvelope"
|
||||
"503":
|
||||
description: No background worker is bound to this process, so the job cannot be queued. The upload was staged and is
|
||||
discarded; retry once a worker is running.
|
||||
description: No background worker is bound to this process, or the queue did not accept the job, so nothing will run it.
|
||||
The staged upload is discarded (and a created job is marked failed); retry once a worker is running.
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
@@ -30819,36 +30827,37 @@ components:
|
||||
anamnesis.fact.invalidValue; auth — auth.admin.required, auth.already_authenticated,
|
||||
auth.mfa.code_invalid, auth.missing, auth.refresh.invalid, auth.refresh.reuse, auth.refresh.revoked,
|
||||
auth.scope.insufficient, auth.stepup.required, auth.token.expired, auth.token.invalid; backup —
|
||||
backup.payload.undecryptable, backup.restore.active, backup.section.missing, backup.upload.too_large;
|
||||
body-site — body-site.invalid; coach — coach.budget.exceeded, coach.conversation.invalidTitle,
|
||||
coach.conversation.notFound, coach.fenced.attachmentConflict, coach.fenced.attachmentLimit,
|
||||
coach.fenced.attachmentNotFound, coach.fenced.attachmentNotIndexed, coach.fenced.attachmentRequired,
|
||||
coach.fenced.attachmentUnavailable, coach.fenced.invalid, coach.request.invalid; coach-prefs —
|
||||
coach-prefs.body.invalid_json, coach-prefs.body.invalid_shape; consent — consent.ai.required; cycle —
|
||||
cycle.bulk.invalid, cycle.bulk.too_large, cycle.calendar.invalid, cycle.calendar.range,
|
||||
cycle.cycles.invalid, cycle.day-log.conflict, cycle.day-log.invalid, cycle.disabled,
|
||||
cycle.period.invalid, cycle.symptom.custom.invalid, cycle.symptom.custom.limit; cycle-prefs —
|
||||
cycle-prefs.body.invalid_json, cycle-prefs.invalid; disable-coach — disable-coach.body.invalid_json;
|
||||
documents — documents.chat.conversationNotFound, documents.chat.invalid,
|
||||
documents.inbound.alreadyConfirmed, documents.inbound.alreadyPartlyConfirmed,
|
||||
documents.inbound.budgetExceeded, documents.inbound.decryptFailed, documents.inbound.encounterNotFound,
|
||||
documents.inbound.episodeNotFound, documents.inbound.extractFailed, documents.inbound.factNotFound,
|
||||
documents.inbound.factNotPending, documents.inbound.factTypeMismatch, documents.inbound.fileTooLarge,
|
||||
documents.inbound.fileType, documents.inbound.invalidMetadata, documents.inbound.invalidQuery,
|
||||
documents.inbound.invalidUpdate, documents.inbound.localOcrDisabled, documents.inbound.notFound,
|
||||
documents.inbound.notIndexed, documents.inbound.pdfNeedsAnthropic,
|
||||
documents.inbound.providerUnsupported, documents.inbound.quotaExceeded, documents.inbound.rateLimited,
|
||||
documents.inbound.restoreDuplicate, documents.inbound.restoreGone, documents.inbound.sourceAliasLimit,
|
||||
documents.inbound.uploadBusy, documents.inbound.uploadTimeout, documents.inbound.vaccinationNotFound;
|
||||
encounter — encounter.invalid, encounter.link.invalid, encounter.practitioner-not-found,
|
||||
encounter.reminder-conflict, encounter.reminder-not-found; environment — environment.invalid,
|
||||
environment.no_home, environment.range_too_large, environment.travel.not_found; export —
|
||||
export.selection.empty, export.selection.unknown_leaf; familyHistory — familyHistory.invalid; feedback —
|
||||
feedback.body.invalid; health_score_config — health_score_config.invalid,
|
||||
health_score_config.too_narrow; illness — illness.day-log.invalid, illness.disabled,
|
||||
illness.episode.chronic-no-resolve, illness.episode.invalid, illness.episode.invalid-window,
|
||||
illness.episode.parent-not-found, illness.insights.invalid; import — import.write_failed; ingest —
|
||||
ingest.rate_limited; insights — insights.generate.budgetExceeded, insights.generate.outboundScreened,
|
||||
backup.foreign_reference, backup.key.missing, backup.payload.undecryptable, backup.restore.active,
|
||||
backup.section.missing, backup.upload.too_large; body-site — body-site.invalid; coach —
|
||||
coach.budget.exceeded, coach.conversation.invalidTitle, coach.conversation.notFound,
|
||||
coach.fenced.attachmentConflict, coach.fenced.attachmentLimit, coach.fenced.attachmentNotFound,
|
||||
coach.fenced.attachmentNotIndexed, coach.fenced.attachmentRequired, coach.fenced.attachmentUnavailable,
|
||||
coach.fenced.invalid, coach.request.invalid; coach-prefs — coach-prefs.body.invalid_json,
|
||||
coach-prefs.body.invalid_shape; consent — consent.ai.required; cycle — cycle.bulk.invalid,
|
||||
cycle.bulk.too_large, cycle.calendar.invalid, cycle.calendar.range, cycle.cycles.invalid,
|
||||
cycle.day-log.conflict, cycle.day-log.invalid, cycle.disabled, cycle.period.invalid,
|
||||
cycle.symptom.custom.invalid, cycle.symptom.custom.limit; cycle-prefs — cycle-prefs.body.invalid_json,
|
||||
cycle-prefs.invalid; disable-coach — disable-coach.body.invalid_json; documents —
|
||||
documents.chat.conversationNotFound, documents.chat.invalid, documents.inbound.alreadyConfirmed,
|
||||
documents.inbound.alreadyPartlyConfirmed, documents.inbound.budgetExceeded,
|
||||
documents.inbound.decryptFailed, documents.inbound.encounterNotFound, documents.inbound.episodeNotFound,
|
||||
documents.inbound.extractFailed, documents.inbound.factNotFound, documents.inbound.factNotPending,
|
||||
documents.inbound.factTypeMismatch, documents.inbound.fileTooLarge, documents.inbound.fileType,
|
||||
documents.inbound.invalidMetadata, documents.inbound.invalidQuery, documents.inbound.invalidUpdate,
|
||||
documents.inbound.localOcrDisabled, documents.inbound.notFound, documents.inbound.notIndexed,
|
||||
documents.inbound.pdfNeedsAnthropic, documents.inbound.providerUnsupported,
|
||||
documents.inbound.quotaExceeded, documents.inbound.rateLimited, documents.inbound.restoreDuplicate,
|
||||
documents.inbound.restoreGone, documents.inbound.sourceAliasLimit, documents.inbound.uploadBusy,
|
||||
documents.inbound.uploadTimeout, documents.inbound.vaccinationNotFound; encounter — encounter.invalid,
|
||||
encounter.link.invalid, encounter.practitioner-not-found, encounter.reminder-conflict,
|
||||
encounter.reminder-not-found; environment — environment.invalid, environment.no_home,
|
||||
environment.range_too_large, environment.travel.not_found; export — export.selection.empty,
|
||||
export.selection.unknown_leaf; familyHistory — familyHistory.invalid; feedback — feedback.body.invalid;
|
||||
health_score_config — health_score_config.invalid, health_score_config.too_narrow; illness —
|
||||
illness.day-log.invalid, illness.disabled, illness.episode.chronic-no-resolve, illness.episode.invalid,
|
||||
illness.episode.invalid-window, illness.episode.parent-not-found, illness.insights.invalid; import —
|
||||
import.apple_health.busy, import.write_failed; ingest — ingest.rate_limited; insights —
|
||||
insights.generate.budgetExceeded, insights.generate.outboundScreened,
|
||||
insights.generate.profileScopeChanged, insights.generate.scopeChanged; labs —
|
||||
labs.biomarker.create.invalid, labs.biomarker.duplicate, labs.biomarker.notFound,
|
||||
labs.biomarker.update.invalid, labs.biomarker.update.referenceRangeInvalid, labs.create.invalid,
|
||||
@@ -72376,21 +72385,26 @@ components:
|
||||
type: string
|
||||
description: "Stable reason: `backup_not_found`, `backup_changed`, `backup.payload.undecryptable`, `schema_invalid`,
|
||||
`incompatible_schema_version`, `owner_mismatch`, `owner_not_found`, `backup.section.missing`,
|
||||
`document_ciphertext_missing`, `time_budget`, `transaction_failed`, `interrupted`, `not_started`,
|
||||
`enqueue_failed`, `failed_after_commit`, `unexpected`. Every code but `failed_after_commit` means the
|
||||
account was not changed."
|
||||
`document_ciphertext_missing`, `backup.key.missing`, `backup.foreign_reference`, `time_budget`,
|
||||
`transaction_failed`, `interrupted`, `not_started`, `enqueue_failed`, `failed_after_commit`, `unexpected`.
|
||||
Every code but `failed_after_commit` means the account was not changed."
|
||||
message:
|
||||
type: string
|
||||
sections:
|
||||
type: array
|
||||
items:
|
||||
type: string
|
||||
keyIds:
|
||||
type: array
|
||||
items:
|
||||
type: string
|
||||
required:
|
||||
- code
|
||||
- message
|
||||
additionalProperties: false
|
||||
description: Why the restore did not happen, in a sentence an operator can act on. `sections` names the missing sections
|
||||
for `backup.section.missing`.
|
||||
for `backup.section.missing`; `keyIds` names the encryption key ids the file needs and this server cannot open
|
||||
it with, for `backup.key.missing`.
|
||||
AdminBackupRestoreJobResponse:
|
||||
type: object
|
||||
properties:
|
||||
|
||||
+101
-26
@@ -16,13 +16,23 @@ container (queue `data-backup-offhost`). Object key layout:
|
||||
|
||||
```
|
||||
magic = "HLBK" (4 bytes, ASCII)
|
||||
version = 0x03 (1 byte)
|
||||
version = 0x04 (1 byte)
|
||||
keyIdLen, keyId (1 byte + 12 ASCII hex: the first 12 hex digits of SHA-256(key))
|
||||
iv = 12 random bytes (AES-GCM nonce)
|
||||
ciphertext = N bytes (AES-256-GCM over gzip(JSON dump), key = BACKUP_ENCRYPTION_KEY)
|
||||
ciphertext = N bytes (AES-256-GCM over gzip(JSON dump), key = BACKUP_ENCRYPTION_KEY,
|
||||
associated data = "healthlog/offhost-backup/v4|<object key>")
|
||||
authTag = 16 bytes (AES-GCM tag, trailing)
|
||||
```
|
||||
|
||||
Three versions exist in the wild and all three restore. `0x01` encrypted the
|
||||
Version `0x04` (from v1.39.3) adds two things. The key id says which off-host
|
||||
key an object needs, so a reader holding a retired key as well finds the right
|
||||
one. The associated data is the object's own key in the bucket,
|
||||
`YYYY-MM-DD/user-<id>.json.enc`: an object copied to another account's name or
|
||||
another date does not decrypt, where before it opened as if it belonged there.
|
||||
The restore script passes the key it downloaded, so nothing changes for an
|
||||
operator.
|
||||
|
||||
Four versions exist in the wild and all four restore. `0x01` encrypted the
|
||||
JSON directly and `0x02` gzipped it first; both carry the tag in FRONT of the
|
||||
ciphertext, which is what made them impossible to write a piece at a time —
|
||||
GCM only produces the tag once the last block is in, so a leading tag means the
|
||||
@@ -33,32 +43,59 @@ the restore script needs no flag to tell them apart.
|
||||
|
||||
## Required env vars
|
||||
|
||||
| Var | Required | Notes |
|
||||
| ----------------------- | -------- | -------------------------------------------------------------------- |
|
||||
| `BACKUP_ENCRYPTION_KEY` | yes | 64 hex chars or 32-byte base64. **Different from `ENCRYPTION_KEY`.** |
|
||||
| `BACKUP_S3_ENDPOINT` | yes | e.g. `https://<account>.r2.cloudflarestorage.com` |
|
||||
| `BACKUP_S3_BUCKET` | yes | |
|
||||
| `BACKUP_S3_ACCESS_KEY` | yes | |
|
||||
| `BACKUP_S3_SECRET_KEY` | yes | |
|
||||
| `BACKUP_S3_REGION` | no | defaults to `auto` (Cloudflare R2) |
|
||||
| `BACKUP_RETENTION_DAYS` | no | not read by the app; the number for your bucket's lifecycle rule |
|
||||
| Var | Required | Notes |
|
||||
| --------------------------------- | -------- | ----------------------------------------------------------------------------------- |
|
||||
| `BACKUP_ENCRYPTION_KEY` | yes | 64 hex chars or 32-byte base64. **Different from `ENCRYPTION_KEY`.** |
|
||||
| `BACKUP_S3_ENDPOINT` | yes | e.g. `https://<account>.r2.cloudflarestorage.com` |
|
||||
| `BACKUP_S3_BUCKET` | yes | |
|
||||
| `BACKUP_S3_ACCESS_KEY` | yes | |
|
||||
| `BACKUP_S3_SECRET_KEY` | yes | |
|
||||
| `BACKUP_S3_REGION` | no | defaults to `auto` (Cloudflare R2) |
|
||||
| `BACKUP_RETENTION_DAYS` | no | not read by the app; the number for your bucket's lifecycle rule |
|
||||
| `BACKUP_ENCRYPTION_PREVIOUS_KEYS` | no | retired off-host keys, comma-separated, read only (see "Rotating the off-host key") |
|
||||
|
||||
## Bucket permissions
|
||||
|
||||
The worker needs `PutObject`, `GetObject` and `AbortMultipartUpload`. It never
|
||||
calls `DeleteObject` on a backup key, so a compromised worker cannot wipe the
|
||||
history; `AbortMultipartUpload` only reaches an upload that same worker started
|
||||
and is what clears the parts of a run that failed halfway. Without it, a failed
|
||||
upload leaves parts that are billed and do not show in a bucket listing. On
|
||||
Cloudflare R2 the **Object Read & Write** token already covers all three.
|
||||
The worker needs `PutObject`, `GetObject`, `ListBucket`, `DeleteObject` and
|
||||
`AbortMultipartUpload`. On Cloudflare R2 the **Object Read & Write** token
|
||||
covers all of them.
|
||||
|
||||
## Bucket lifecycle (recommended)
|
||||
`DeleteObject` is needed because the app deletes objects in two cases, and
|
||||
only those:
|
||||
|
||||
The worker never deletes a backup object: its grant covers PutObject,
|
||||
GetObject and AbortMultipartUpload only, so a compromised worker cannot
|
||||
wipe the history, and nothing in the app reads `BACKUP_RETENTION_DAYS`.
|
||||
Expiry is the storage provider's lifecycle rule, set to match the number
|
||||
you wrote there:
|
||||
- **An account is deleted, or its data is wiped.** Deleting an account,
|
||||
deleting a managed profile and "Delete all data" each write a deletion
|
||||
request in the same transaction as the deletion. The `offhost-backup-purge` job lists the bucket, deletes every
|
||||
`YYYY-MM-DD/user-<id>.json.enc` of that account and removes the request once
|
||||
nothing of the account is left. It runs right after the deletion and again
|
||||
every night at 03:40. A copy the nightly upload was writing while the account
|
||||
was deleted is removed by the upload itself.
|
||||
- **The connection test** deletes its own `_healthcheck/<ts>.bin`.
|
||||
|
||||
The admin wipe of every account deletes nothing in the bucket. It is you
|
||||
clearing your own host, the bucket is yours, and its copies are the way back
|
||||
from a wipe pressed by mistake. If you are handing the host on, empty the
|
||||
bucket yourself.
|
||||
|
||||
Earlier versions of this page said the worker never deletes an object. That
|
||||
was the design until v1.39.3, and it meant a deleted account's copies stayed
|
||||
in the bucket for as long as the lifecycle rule kept them, or forever without
|
||||
one. If the bucket refuses the delete (a credential without `DeleteObject`),
|
||||
the request stays, the job retries it every night, and **Admin → Backups →
|
||||
Off-host** shows how many deletions are outstanding and the bucket's answer.
|
||||
|
||||
`AbortMultipartUpload` only reaches an upload the same worker started and is
|
||||
what clears the parts of a run that failed halfway. Without it, a failed
|
||||
upload leaves parts that are billed and do not show in a bucket listing.
|
||||
|
||||
## Bucket lifecycle (required for retention)
|
||||
|
||||
The app does not expire objects by age, and nothing in the app reads
|
||||
`BACKUP_RETENTION_DAYS`. Expiry is the storage provider's lifecycle rule, set
|
||||
to match the number you wrote there. Without one, every nightly copy of every
|
||||
account is kept forever. **Admin → Backups → Off-host** reads the bucket's
|
||||
lifecycle configuration and says so when there is none, or when the credential
|
||||
may not read it (grant `GetLifecycleConfiguration` to let it check):
|
||||
|
||||
```
|
||||
Filter: "" (all objects)
|
||||
@@ -343,6 +380,28 @@ in seconds having backed up nothing is that defect and not your record.
|
||||
|
||||
### Key rotation
|
||||
|
||||
Rotation reaches a stored backup's envelope and nothing inside it. The
|
||||
content of a disaster-recovery backup is the database's ciphertext, carried
|
||||
as it is stored, and a restore writes it back verbatim: a note, a document or
|
||||
a coach message inside a copy taken before a rotation still needs the key the
|
||||
rotation retired. The same holds for every copy in the off-host bucket and for
|
||||
every file an operator has downloaded. So the rule is: **keep a retired
|
||||
`ENCRYPTION_KEYS` entry until no backup you may still restore needs it.**
|
||||
**Admin → Encryption → Keys the backups still need** lists, per key id, the
|
||||
stored copies that need it with the oldest date, and when the last off-host
|
||||
copy needing it expires under the bucket's lifecycle rule. Copies written
|
||||
before v1.39.3 did not record their keys and are listed as such.
|
||||
|
||||
Every restore, the restore preview, the admin upload and the monthly drill
|
||||
check this before anything is changed: a file whose content needs a key this
|
||||
server does not have (or an id whose key material differs) is refused with the
|
||||
key id named, and the account is left as it was.
|
||||
|
||||
Re-encrypting the content of stored backups during rotation was considered and
|
||||
not done: it would mean decrypting, parsing, re-encrypting and re-writing every
|
||||
copy in full, and it still could not reach the off-host copies or downloaded
|
||||
files, so the rule above would be needed regardless.
|
||||
|
||||
`scripts/rotate-encryption-key.ts` and the rotation in the admin console
|
||||
re-seal every piece under the active key, in small batches, without reading
|
||||
the backup inside. The link between a piece, its copy and its position is
|
||||
@@ -367,6 +426,17 @@ read back. Restoring Monday's copy on Wednesday therefore costs the account
|
||||
everything it recorded on Tuesday. If the current state is worth keeping, take
|
||||
a fresh snapshot from the backups console before restoring the old one.
|
||||
|
||||
## Rotating the off-host key
|
||||
|
||||
`BACKUP_ENCRYPTION_KEY` is the key new objects are written under. To rotate
|
||||
it, set the new key there and move the old one into
|
||||
`BACKUP_ENCRYPTION_PREVIOUS_KEYS` (comma-separated if there are several), then
|
||||
restart. New objects are written under the new key; older objects stay
|
||||
readable because every version-4 object names its key and older objects are
|
||||
tried against each key in turn. Remove the old key once the lifecycle rule has
|
||||
retired every object written under it, that is `BACKUP_RETENTION_DAYS` after
|
||||
the restart.
|
||||
|
||||
## Monthly restore drill (automatic)
|
||||
|
||||
Since v1.16.4 a pg-boss job (`data-restore-drill`, cron `11 4 1 * *` —
|
||||
@@ -386,8 +456,13 @@ Outcomes:
|
||||
- **Failure** — empty bucket, fetch error, decryption failure (wrong or
|
||||
rotated key), malformed JSON: pages the same way. A decryption
|
||||
failure right after a `BACKUP_ENCRYPTION_KEY` change means the new
|
||||
key cannot read the existing objects — re-encrypt or accept that
|
||||
pre-rotation backups are only readable with the retired key.
|
||||
key cannot read the existing objects: put the old key into
|
||||
`BACKUP_ENCRYPTION_PREVIOUS_KEYS` (see "Rotating the off-host key").
|
||||
- **Content under a missing application key** — the object opens, but its
|
||||
content was written under an `ENCRYPTION_KEYS` entry this server no longer
|
||||
has, or under an id whose key material changed. The drill decrypts one value
|
||||
per key id it finds, and pages naming the key. A restore of that object
|
||||
would write back rows nobody can open; put the key back before restoring.
|
||||
- **Not configured** — deployments without the `BACKUP_S3_*` vars skip
|
||||
silently (wide-event warning only).
|
||||
|
||||
|
||||
@@ -99,14 +99,34 @@ under `v1` (the synthetic id assigned to the existing `ENCRYPTION_KEY`).
|
||||
Restart. The legacy single-key fallback is now disconnected; only `v2`
|
||||
exists.
|
||||
|
||||
> **Backups are covered, and were not always.** `DataBackup.data` holds every
|
||||
> weekly disaster-recovery snapshot and every uploaded pack, encrypted like
|
||||
> everything else but under a column called `data`. It was outside the
|
||||
> registry until v1.38.6, so a rotation before that release reported zero
|
||||
> rows remaining without ever reading a backup. If you rotated on an older
|
||||
> release and dropped the previous key, the stored backups are encrypted
|
||||
> under the key you removed: put that key back into `ENCRYPTION_KEYS` and
|
||||
> re-run the rotation on this release before removing it again.
|
||||
> **The rows in the database are covered. The content of a backup is not.**
|
||||
> Rotation re-encrypts every registered column, including `DataBackup.data`
|
||||
> and the pieces in `DataBackupChunk`, so every stored copy opens under the
|
||||
> new key. What it cannot change is what a disaster-recovery backup carries
|
||||
> inside: the database's ciphertext as it was stored when the copy was taken,
|
||||
> which a restore writes back verbatim. A copy taken before the rotation still
|
||||
> needs the old key for its notes, documents and coach history, and so does
|
||||
> every copy in the off-host bucket and every backup file you downloaded.
|
||||
>
|
||||
> So before step 4, also check **Admin → Encryption → Keys the backups still
|
||||
> need**. It lists, per key id, the stored copies that need it with the oldest
|
||||
> date, and when the last off-host copy needing it expires under the bucket's
|
||||
> lifecycle rule. Keep the old key in `ENCRYPTION_KEYS` until it is no longer
|
||||
> listed there (the weekly copy replaces itself within a week; delete old
|
||||
> uploaded copies you no longer want, and wait out the off-host retention), or
|
||||
> accept that those copies cannot be restored. Copies written before v1.39.3
|
||||
> did not record their keys and are listed as such: treat them as needing
|
||||
> every key that existed when they were written.
|
||||
>
|
||||
> Dropping the key early no longer fails silently: a restore, a restore
|
||||
> preview or an upload of a copy that needs a missing key is refused with the
|
||||
> key id named, and nothing is changed. Putting the key back makes the copy
|
||||
> restorable again.
|
||||
>
|
||||
> If you rotated on a release before v1.38.6 and dropped the previous key,
|
||||
> the stored backups themselves are encrypted under the key you removed: put
|
||||
> that key back into `ENCRYPTION_KEYS` and re-run the rotation on this release
|
||||
> before removing it again.
|
||||
|
||||
## Adding a third key (v2 → v3)
|
||||
|
||||
|
||||
+1
-1
@@ -6579,7 +6579,7 @@
|
||||
"delete": {
|
||||
"title": "Löschen oder zurücksetzen",
|
||||
"description": "Lösche dein Konto vollständig oder lösche deine Gesundheitsdaten und behalte das Konto.",
|
||||
"backupLag": "Die Löschung entfernt deine Daten sofort. Verschlüsselte Off-Host-Backups enthalten sie weiterhin, bis sie aus dem Aufbewahrungsfenster des Betreibers herausfallen.",
|
||||
"backupLag": "Die Löschung entfernt deine Daten sofort. Wenn dieser Server verschlüsselte externe Sicherungen anlegt, werden die Kopien deiner Daten gleich danach aus diesem Speicher gelöscht, oder in der nächsten Nacht, falls der Speicher nicht erreichbar ist.",
|
||||
"openAdvanced": "Löschen oder zurücksetzen"
|
||||
},
|
||||
"posture": {
|
||||
|
||||
+1
-1
@@ -6579,7 +6579,7 @@
|
||||
"delete": {
|
||||
"title": "Delete or reset",
|
||||
"description": "Remove your account entirely, or wipe your health records while keeping the account.",
|
||||
"backupLag": "Deletion removes your data immediately. Encrypted off-host backups still contain it until they roll off the operator's backup retention window.",
|
||||
"backupLag": "Deletion removes your data immediately. If this server keeps encrypted off-host backups, the copies of your data are deleted from that storage right after, or on the next nightly run if the storage is unreachable.",
|
||||
"openAdvanced": "Delete or reset"
|
||||
},
|
||||
"posture": {
|
||||
|
||||
+1
-1
@@ -6579,7 +6579,7 @@
|
||||
"delete": {
|
||||
"title": "Eliminar o restablecer",
|
||||
"description": "Elimina tu cuenta por completo o borra tus registros de salud conservando la cuenta.",
|
||||
"backupLag": "La eliminación borra tus datos de inmediato. Las copias de seguridad cifradas externas aún los contienen hasta que caduquen según la ventana de retención del operador.",
|
||||
"backupLag": "La eliminación borra tus datos de inmediato. Si este servidor guarda copias de seguridad cifradas externas, las copias de tus datos se borran de ese almacenamiento justo después, o en la siguiente ejecución nocturna si no está disponible.",
|
||||
"openAdvanced": "Eliminar o restablecer"
|
||||
},
|
||||
"posture": {
|
||||
|
||||
+1
-1
@@ -6579,7 +6579,7 @@
|
||||
"delete": {
|
||||
"title": "Supprimer ou réinitialiser",
|
||||
"description": "Supprimez complètement votre compte, ou effacez vos données de santé tout en conservant le compte.",
|
||||
"backupLag": "La suppression efface vos données immédiatement. Les sauvegardes chiffrées hors site les contiennent encore jusqu'à leur expiration selon la fenêtre de conservation de l'opérateur.",
|
||||
"backupLag": "La suppression efface vos données immédiatement. Si ce serveur conserve des sauvegardes externes chiffrées, les copies de vos données sont supprimées de ce stockage juste après, ou lors du prochain passage nocturne s'il est injoignable.",
|
||||
"openAdvanced": "Supprimer ou réinitialiser"
|
||||
},
|
||||
"posture": {
|
||||
|
||||
+1
-1
@@ -6579,7 +6579,7 @@
|
||||
"delete": {
|
||||
"title": "Elimina o reimposta",
|
||||
"description": "Elimina completamente il tuo account oppure cancella i tuoi dati sanitari mantenendo l'account.",
|
||||
"backupLag": "L'eliminazione rimuove subito i tuoi dati. I backup cifrati esterni li contengono ancora finché non scadono secondo la finestra di conservazione dell'operatore.",
|
||||
"backupLag": "L'eliminazione rimuove subito i tuoi dati. Se questo server conserva backup esterni cifrati, le copie dei tuoi dati vengono eliminate da quello spazio subito dopo, o alla successiva esecuzione notturna se non è raggiungibile.",
|
||||
"openAdvanced": "Elimina o reimposta"
|
||||
},
|
||||
"posture": {
|
||||
|
||||
+1
-1
@@ -6579,7 +6579,7 @@
|
||||
"delete": {
|
||||
"title": "삭제 또는 초기화",
|
||||
"description": "계정을 완전히 없애거나, 계정은 두고 건강 기록만 지워요.",
|
||||
"backupLag": "삭제하면 데이터는 즉시 지워져요. 외부에 보관된 암호화 백업에는 운영자의 백업 보관 기간이 끝날 때까지 남아 있어요.",
|
||||
"backupLag": "삭제하면 데이터는 즉시 지워져요. 이 서버가 암호화된 외부 백업을 보관한다면 그 저장소에 있는 데이터 사본도 곧바로 삭제되고, 저장소에 연결할 수 없으면 다음 야간 실행 때 삭제돼요.",
|
||||
"openAdvanced": "삭제 또는 초기화"
|
||||
},
|
||||
"posture": {
|
||||
|
||||
+1
-1
@@ -6579,7 +6579,7 @@
|
||||
"delete": {
|
||||
"title": "Usuń lub zresetuj",
|
||||
"description": "Usuń całe konto albo wyczyść dane zdrowotne, zachowując konto.",
|
||||
"backupLag": "Usunięcie natychmiast usuwa Twoje dane. Zaszyfrowane kopie zapasowe poza serwerem nadal je zawierają, dopóki nie wygasną zgodnie z oknem retencji operatora.",
|
||||
"backupLag": "Usunięcie natychmiast usuwa Twoje dane. Jeśli ten serwer przechowuje zaszyfrowane kopie zapasowe poza serwerem, kopie Twoich danych są usuwane z tego magazynu zaraz potem, a jeśli jest niedostępny, przy następnym nocnym przebiegu.",
|
||||
"openAdvanced": "Usuń lub zresetuj"
|
||||
},
|
||||
"posture": {
|
||||
|
||||
@@ -5183,7 +5183,7 @@ model OffhostBackupKeyUse {
|
||||
model OffhostPurgeRequest {
|
||||
id String @id @default(cuid())
|
||||
subjectId String @map("subject_id")
|
||||
/// `account_deleted`, `data_wiped`, `managed_profile_deleted` or `admin_wipe`.
|
||||
/// `account_deleted`, `data_wiped` or `managed_profile_deleted`.
|
||||
reason String
|
||||
requestedAt DateTime @default(now()) @map("requested_at")
|
||||
attempts Int @default(0)
|
||||
|
||||
@@ -1,8 +1,3 @@
|
||||
import {
|
||||
kickOffhostPurge,
|
||||
PURGE_ALL_SUBJECTS,
|
||||
requestOffhostPurge,
|
||||
} from "@/lib/jobs/offhost-purge";
|
||||
import { prisma } from "@/lib/db";
|
||||
import { apiHandler, requireAdmin } from "@/lib/api-handler";
|
||||
import { auditLog } from "@/lib/auth/audit";
|
||||
@@ -113,9 +108,6 @@ export const DELETE = apiHandler(async (request: NextRequest) => {
|
||||
// columns does not. Same classification contract as the model list.
|
||||
await tx.user.updateMany({ data: USER_RESET });
|
||||
|
||||
// Every account's nightly off-host copies go too.
|
||||
await requestOffhostPurge(tx, PURGE_ALL_SUBJECTS, "admin_wipe");
|
||||
|
||||
return perModel;
|
||||
},
|
||||
{
|
||||
@@ -146,7 +138,6 @@ export const DELETE = apiHandler(async (request: NextRequest) => {
|
||||
// v1.16.9 — the wipe touched every user's rows; clear every cache
|
||||
// bucket so no per-user payload survives the reset.
|
||||
invalidateAllCaches();
|
||||
await kickOffhostPurge();
|
||||
|
||||
return apiSuccess({ cleared: true, deletedRows, models: counts });
|
||||
});
|
||||
@@ -223,9 +223,9 @@ export const WIPE_MODELS = [
|
||||
// ended. About this account's record, so they go with it.
|
||||
"BackupRestoreJob",
|
||||
// What this host last put in the operator's bucket for this account. The
|
||||
// objects themselves stay — the worker holds no DeleteObject grant and the
|
||||
// bucket's lifecycle rule owns their retirement — but the row saying when
|
||||
// they were written is about this account and goes with it.
|
||||
// objects themselves are deleted by the purge request the wipe writes
|
||||
// (`requestOffhostPurge`); this row, saying when they were written, is about
|
||||
// this account and goes with it.
|
||||
"OffhostBackupState",
|
||||
"AuditLog",
|
||||
"ConsentReceipt",
|
||||
|
||||
@@ -30,12 +30,7 @@
|
||||
*/
|
||||
import { Buffer } from "node:buffer";
|
||||
|
||||
import {
|
||||
decrypt,
|
||||
decryptBytes,
|
||||
extractKeyIdFromBytes,
|
||||
getConfiguredKeyIds,
|
||||
} from "@/lib/crypto";
|
||||
import { decrypt, decryptBytes, getConfiguredKeyIds } from "@/lib/crypto";
|
||||
|
||||
/** The key id a value with no key id prefix was written under. */
|
||||
export const LEGACY_INNER_KEY_ID = "v1";
|
||||
|
||||
@@ -193,7 +193,8 @@ describe("extractExportXml", () => {
|
||||
|
||||
await expect(extractExportXml(zipPath)).rejects.toThrow(/zip bomb/);
|
||||
expect(staged().filter((n) => !before.has(n))).toEqual([]);
|
||||
});
|
||||
// Inflating past the 64 MiB floor takes a moment on a loaded runner.
|
||||
}, 60_000);
|
||||
|
||||
it("throws when the export.xml member is missing", async () => {
|
||||
const payload = Buffer.from("noop");
|
||||
|
||||
@@ -13,11 +13,7 @@ import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
vi.mock("@/lib/auth/audit", () => ({ auditLog: vi.fn() }));
|
||||
|
||||
import { auditLog } from "@/lib/auth/audit";
|
||||
import {
|
||||
processOffhostPurges,
|
||||
PURGE_ALL_SUBJECTS,
|
||||
requestOffhostPurge,
|
||||
} from "../offhost-purge";
|
||||
import { processOffhostPurges, requestOffhostPurge } from "../offhost-purge";
|
||||
|
||||
interface Row {
|
||||
id: string;
|
||||
@@ -175,18 +171,6 @@ describe("off-host purge", () => {
|
||||
);
|
||||
});
|
||||
|
||||
it("clears every account object for the admin wipe", async () => {
|
||||
const { prisma } = makeLedger();
|
||||
await requestOffhostPurge(
|
||||
prisma as never,
|
||||
PURGE_ALL_SUBJECTS,
|
||||
"admin_wipe",
|
||||
);
|
||||
const bucket = makeBucket(BUCKET);
|
||||
await processOffhostPurges(prisma as never, bucket);
|
||||
expect([...bucket.store]).toEqual(["_healthcheck/1.bin"]);
|
||||
});
|
||||
|
||||
it("records nothing on a host without off-host backup", async () => {
|
||||
vi.stubEnv("BACKUP_S3_ENDPOINT", "");
|
||||
const { rows, prisma } = makeLedger();
|
||||
|
||||
@@ -592,21 +592,21 @@ export async function handleAppleHealthImport(
|
||||
// left behind in that case either.
|
||||
if (extractedXmlPath) safeUnlink(extractedXmlPath);
|
||||
safeUnlink(uploadPath);
|
||||
await prisma.importJob
|
||||
.update({
|
||||
try {
|
||||
await prisma.importJob.update({
|
||||
where: { id: importJobId },
|
||||
data: {
|
||||
status: "failed",
|
||||
failureReason: reason.slice(0, 1000),
|
||||
completedAt: new Date(),
|
||||
},
|
||||
})
|
||||
.catch((updateErr: unknown) => {
|
||||
console.warn(
|
||||
`[apple-health-import] Could not record the failure on ImportJob=${importJobId}`,
|
||||
updateErr,
|
||||
);
|
||||
});
|
||||
} catch (updateErr) {
|
||||
console.warn(
|
||||
`[apple-health-import] Could not record the failure on ImportJob=${importJobId}`,
|
||||
updateErr,
|
||||
);
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -11,15 +11,15 @@
|
||||
* Object key layout:
|
||||
* <bucket>/<YYYY-MM-DD>/user-<userId>.json.enc
|
||||
*
|
||||
* Retention: the worker NEVER calls DeleteObject on backup keys. Operators
|
||||
* MUST configure a bucket-level lifecycle rule (e.g. expire after
|
||||
* `BACKUP_RETENTION_DAYS`, which this module reads nowhere — see
|
||||
* `loadOffhostConfig`). This keeps the IAM grant for the worker
|
||||
* limited to PutObject + GetObject + AbortMultipartUpload, so a compromised
|
||||
* worker cannot wipe the backup history. The abort is what cleans up a run
|
||||
* that failed partway rather than leaving billed, unlistable parts behind;
|
||||
* `AbortMultipartUpload` can only touch an upload this worker started, never
|
||||
* a finished object. See docs/ops/backup-restore.md.
|
||||
* Retention: the worker does not expire objects by age. Operators configure a
|
||||
* bucket-level lifecycle rule (e.g. expire after `BACKUP_RETENTION_DAYS`,
|
||||
* which this module reads nowhere — see `loadOffhostConfig`); the admin
|
||||
* off-host card reads that rule back (`probeOffhostLifecycle`) and says when
|
||||
* there is none. Objects ARE deleted in one case: an account that is deleted
|
||||
* or whose data is wiped (`offhost-purge.ts`), which needs DeleteObject in the
|
||||
* grant. `AbortMultipartUpload` cleans up a run that failed partway rather
|
||||
* than leaving billed, unlistable parts behind. See
|
||||
* docs/ops/backup-restore.md.
|
||||
*/
|
||||
import { Buffer } from "node:buffer";
|
||||
import {
|
||||
@@ -108,8 +108,8 @@ export function loadOffhostConfig(): OffhostBackupConfig | null {
|
||||
// `BACKUP_RETENTION_DAYS` is deliberately absent from this config. It used
|
||||
// to be parsed and clamped here and then read by nobody, which read as an
|
||||
// enforcer the worker is not: retention belongs to the bucket's lifecycle
|
||||
// rule, and no consumer can ever appear here because DeleteObject is kept
|
||||
// out of the worker's grant on purpose (see the header). The variable stays
|
||||
// rule, which the worker leaves to the bucket rather than deleting by age
|
||||
// (see the header). The variable stays
|
||||
// documented and on the compose whitelist because it is the number the
|
||||
// operator sets that rule to.
|
||||
return {
|
||||
@@ -964,7 +964,7 @@ export async function runOffhostBackup(
|
||||
const purgedMeanwhile =
|
||||
(await prisma.offhostPurgeRequest.count({
|
||||
where: {
|
||||
subjectId: { in: [user.id, "*"] },
|
||||
subjectId: user.id,
|
||||
requestedAt: { gte: accountStartedAt },
|
||||
},
|
||||
})) > 0 || (await prisma.user.count({ where: { id: user.id } })) === 0;
|
||||
|
||||
@@ -17,9 +17,11 @@
|
||||
* the job runs again every night and after every nightly upload, so a late
|
||||
* copy the upload wrote for an account deleted while it ran is caught too.
|
||||
*
|
||||
* `subjectId` "*" is the admin wipe of every account: every account object
|
||||
* goes. The health-check probes and anything else in the bucket that is not
|
||||
* an account object is never touched.
|
||||
* The admin wipe of every account writes no request, on purpose: it is the
|
||||
* operator clearing their own host, the bucket is theirs, and its copies are
|
||||
* the documented way back from a wipe pressed by mistake. The health-check
|
||||
* probes and anything else in the bucket that is not an account object is
|
||||
* never touched.
|
||||
*/
|
||||
import type { Job } from "pg-boss";
|
||||
|
||||
@@ -41,11 +43,8 @@ export const OFFHOST_PURGE_QUEUE = "offhost-backup-purge";
|
||||
/** 03:40, after the nightly upload (02:30) has had its hour. */
|
||||
export const OFFHOST_PURGE_CRON = "40 3 * * *";
|
||||
|
||||
/** Every subject, for the admin wipe of every account. */
|
||||
export const PURGE_ALL_SUBJECTS = "*";
|
||||
|
||||
export type OffhostPurgeReason =
|
||||
"account_deleted" | "data_wiped" | "managed_profile_deleted" | "admin_wipe";
|
||||
"account_deleted" | "data_wiped" | "managed_profile_deleted";
|
||||
|
||||
/** An account object: `<date>/user-<id>.json.enc`. */
|
||||
const ACCOUNT_OBJECT = /^\d{4}-\d{2}-\d{2}\/user-(.+)\.json\.enc$/;
|
||||
@@ -163,14 +162,13 @@ export async function processOffhostPurges(
|
||||
return report;
|
||||
}
|
||||
|
||||
const everything = requests.some((r) => r.subjectId === PURGE_ALL_SUBJECTS);
|
||||
const subjects = new Set(requests.map((r) => r.subjectId));
|
||||
const bySubject = new Map<string, string[]>();
|
||||
for (const { key } of objects) {
|
||||
const match = ACCOUNT_OBJECT.exec(key);
|
||||
if (!match) continue;
|
||||
const subject = match[1];
|
||||
if (!everything && !subjects.has(subject)) continue;
|
||||
if (!subjects.has(subject)) continue;
|
||||
const list = bySubject.get(subject) ?? [];
|
||||
list.push(key);
|
||||
bySubject.set(subject, list);
|
||||
@@ -201,20 +199,12 @@ export async function processOffhostPurges(
|
||||
}
|
||||
|
||||
for (const request of requests) {
|
||||
const failure =
|
||||
request.subjectId === PURGE_ALL_SUBJECTS
|
||||
? failedSubjects.size > 0
|
||||
? [...failedSubjects.values()][0]
|
||||
: null
|
||||
: failedSubjects.get(request.subjectId);
|
||||
const failure = failedSubjects.get(request.subjectId);
|
||||
if (failure) {
|
||||
await markFailed([request.id], failure);
|
||||
continue;
|
||||
}
|
||||
const deleted =
|
||||
request.subjectId === PURGE_ALL_SUBJECTS
|
||||
? [...bySubject.values()].reduce((sum, keys) => sum + keys.length, 0)
|
||||
: (bySubject.get(request.subjectId)?.length ?? 0);
|
||||
const deleted = bySubject.get(request.subjectId)?.length ?? 0;
|
||||
await prisma.offhostPurgeRequest.delete({ where: { id: request.id } });
|
||||
report.completed++;
|
||||
// The account a deleted subject named is gone, and so is its audit
|
||||
|
||||
Reference in new issue
Block a user