diff --git a/docs/ops/backup-restore.md b/docs/ops/backup-restore.md index 4d0a70f0e..ea7ce290c 100644 --- a/docs/ops/backup-restore.md +++ b/docs/ops/backup-restore.md @@ -441,10 +441,12 @@ the restart. Since v1.16.4 a pg-boss job (`data-restore-drill`, cron `11 4 1 * *` — 04:11 on the 1st of each month) exercises the read path end-to-end: -fetch the most recent backup object from the bucket, decrypt it under -the current `BACKUP_ENCRYPTION_KEY`, JSON-parse it, and sanity-check -the payload shape. It performs **no database restore** — it validates -the artefact, not the import path. +fetch every backup object of the most recent date from the bucket (one per +account), decrypt each under the current `BACKUP_ENCRYPTION_KEY`, JSON-parse +it, and sanity-check the payload shape. It performs **no database restore** — +it validates the artefacts, not the import path. Each account is checked and +reported on its own: one object that fails is named, and the others are still +read. Outcomes: @@ -454,14 +456,16 @@ Outcomes: uploader has stalled (or the lifecycle rule is too aggressive). The drill pages via the worker error reporter (stderr + GlitchTip). - **Failure** — empty bucket, fetch error, decryption failure (wrong or - rotated key), malformed JSON: pages the same way. A decryption + rotated key), malformed JSON: pages the same way, once per failing object, + with the object key and how many accounts were checked. A decryption failure right after a `BACKUP_ENCRYPTION_KEY` change means the new key cannot read the existing objects: put the old key into `BACKUP_ENCRYPTION_PREVIOUS_KEYS` (see "Rotating the off-host key"). - **Content under a missing application key** — the object opens, but its content was written under an `ENCRYPTION_KEYS` entry this server no longer - has, or under an id whose key material changed. The drill decrypts one value - per key id it finds, and pages naming the key. A restore of that object + has, or under an id whose key material changed. The drill decrypts the + shortest value of each section under every key id it finds, and pages naming + the key when none of them opens. A restore of that object would write back rows nobody can open; put the key back before restoring. - **Not configured** — deployments without the `BACKUP_S3_*` vars skip silently (wide-event warning only). diff --git a/src/lib/jobs/__tests__/restore-drill.test.ts b/src/lib/jobs/__tests__/restore-drill.test.ts index 41e1cc676..d223c5a45 100644 --- a/src/lib/jobs/__tests__/restore-drill.test.ts +++ b/src/lib/jobs/__tests__/restore-drill.test.ts @@ -93,11 +93,14 @@ describe("runRestoreDrill", () => { "_healthcheck/123.bin": Buffer.from([0x42]), }); const report = await runRestoreDrill(s3, new Date("2026-06-02T04:11:00Z")); - expect(report.objectKey).toBe("2026-06-01/user-abc.json.enc"); + expect(report.accounts.map((a) => a.objectKey)).toEqual([ + "2026-06-01/user-abc.json.enc", + ]); + expect(report.failed).toEqual([]); expect(report.dateKey).toBe("2026-06-01"); expect(report.ageDays).toBe(1); expect(report.stale).toBe(false); - expect(report.recordCounts).toEqual({ + expect(report.accounts[0].recordCounts).toEqual({ measurements: 2, medications: 1, intakeEvents: 0, @@ -109,6 +112,30 @@ describe("runRestoreDrill", () => { expect(s3.deleteObject).not.toHaveBeenCalled(); }); + it("checks every account of the newest date and names the one that fails", async () => { + const tampered = Buffer.from(backupObject({ userId: "user-aaa" })); + tampered[tampered.length - 1] ^= 0xff; + const s3 = makeS3Mock({ + "2026-05-31/user-old.json.enc": backupObject(), + "2026-06-01/user-aaa.json.enc": tampered, + "2026-06-01/user-mmm.json.enc": backupObject({ userId: "user-mmm" }), + // The alphabetically last object, the only one the drill used to read. + "2026-06-01/user-zzz.json.enc": backupObject({ userId: "user-zzz" }), + }); + const report = await runRestoreDrill(s3, new Date("2026-06-02T04:11:00Z")); + expect(report.accounts.map((a) => [a.objectKey, a.ok])).toEqual([ + ["2026-06-01/user-aaa.json.enc", false], + ["2026-06-01/user-mmm.json.enc", true], + ["2026-06-01/user-zzz.json.enc", true], + ]); + expect(report.failed.map((a) => a.objectKey)).toEqual([ + "2026-06-01/user-aaa.json.enc", + ]); + expect(s3.getObject).not.toHaveBeenCalledWith( + "2026-05-31/user-old.json.enc", + ); + }); + it("flags the report stale when the newest backup is older than the threshold", async () => { const s3 = makeS3Mock({ "2026-05-20/user-abc.json.enc": backupObject() }); const report = await runRestoreDrill(s3, new Date("2026-06-01T04:11:00Z")); @@ -132,13 +159,16 @@ describe("runRestoreDrill", () => { }); const now = new Date("2026-06-02T04:11:00Z"); const report = await runRestoreDrill(s3, now); - expect(report.innerKeyIds).toEqual(["old"]); + expect(report.accounts[0].innerKeyIds).toEqual(["old"]); + expect(report.failed).toEqual([]); + const failure = async () => + (await runRestoreDrill(s3, now)).failed.map((a) => a.error).join("\n"); // The rotation ran, and the operator dropped the old key. vi.stubEnv("ENCRYPTION_KEYS", JSON.stringify({ cur: "22".repeat(32) })); vi.stubEnv("ENCRYPTION_ACTIVE_KEY_ID", "cur"); _resetCryptoCacheForTests(); - await expect(runRestoreDrill(s3, now)).rejects.toThrow(/'old'/); + expect(await failure()).toMatch(/'old'/); // Same id, different key material behind it. vi.stubEnv( @@ -146,9 +176,7 @@ describe("runRestoreDrill", () => { JSON.stringify({ old: "33".repeat(32), cur: "22".repeat(32) }), ); _resetCryptoCacheForTests(); - await expect(runRestoreDrill(s3, now)).rejects.toThrow( - /does not open the values/, - ); + expect(await failure()).toMatch(/does not open the values/); vi.unstubAllEnvs(); _resetCryptoCacheForTests(); }); @@ -158,21 +186,25 @@ describe("runRestoreDrill", () => { await expect(runRestoreDrill(s3)).rejects.toThrow(/no backup objects/); }); - it("throws when the object cannot be decrypted (wrong key / tampering)", async () => { + it("fails the account whose object cannot be decrypted (wrong key / tampering)", async () => { const tampered = Buffer.from(backupObject()); tampered[tampered.length - 1] ^= 0xff; const s3 = makeS3Mock({ "2026-06-01/user-abc.json.enc": tampered }); - await expect(runRestoreDrill(s3)).rejects.toThrow(); + const report = await runRestoreDrill(s3); + expect(report.failed.map((a) => a.objectKey)).toEqual([ + "2026-06-01/user-abc.json.enc", + ]); }); - it("throws when the payload is missing core fields", async () => { + it("fails the account whose payload is missing core fields", async () => { const s3 = makeS3Mock({ "2026-06-01/user-abc.json.enc": encryptBackup( JSON.stringify({ exportedAt: "2026-06-01T02:30:00.000Z" }), KEY, ), }); - await expect(runRestoreDrill(s3)).rejects.toThrow(/missing core fields/); + const report = await runRestoreDrill(s3); + expect(report.failed[0].error).toMatch(/missing core fields/); }); it("throws OffhostBackupNotConfiguredError when the S3 vars are unset", async () => { diff --git a/src/lib/jobs/job-outcome.ts b/src/lib/jobs/job-outcome.ts index c66956a88..2ef5e5cc0 100644 --- a/src/lib/jobs/job-outcome.ts +++ b/src/lib/jobs/job-outcome.ts @@ -190,7 +190,10 @@ export const JOB_FACT_ALLOWLIST: ReadonlySet = new Set([ // v1.39.1 — the background restore of a stored backup, and the sweep that // re-queues one a stopped worker left running. Counts and a claim flag. "restore_claimed", + "restore_drill_accounts", + "restore_drill_accounts_failed", "restore_drill_age_days", + "restore_drill_date", "restore_drill_ciphertext_bytes", "restore_drill_intake_events", "restore_drill_measurements", diff --git a/src/lib/jobs/restore-drill.ts b/src/lib/jobs/restore-drill.ts index efd0baf6e..caec85d00 100644 --- a/src/lib/jobs/restore-drill.ts +++ b/src/lib/jobs/restore-drill.ts @@ -9,9 +9,10 @@ * rotation mishap or a bucket-side corruption would only surface on * the day a real restore is needed. * - * This drill closes that gap once a month: fetch the most recent - * backup object, decrypt it, JSON-parse it, and sanity-check the - * payload shape. It deliberately performs NO database restore — the + * This drill closes that gap once a month: fetch every backup object of + * the most recent date (one per account), decrypt it, JSON-parse it, + * sanity-check the payload shape, and open inner values under each + * application key it needs. Each account is reported on its own. It deliberately performs NO database restore — the * drill validates the artefact, not the import path. The result is * surfaced through the wide-event meta on success and through * `reportWorkerError` (stderr + GlitchTip) on any failure, so a broken @@ -53,11 +54,12 @@ const MAX_BACKUP_AGE_DAYS = 3; const BACKUP_KEY_PATTERN = /^(\d{4}-\d{2}-\d{2})\/user-.+\.json\.enc$/; -export interface RestoreDrillReport { +/** What the drill found for one account's object. */ +export interface RestoreDrillAccount { objectKey: string; - dateKey: string; - ageDays: number; - stale: boolean; + ok: boolean; + /** Why the object could not be restored on this server; set when not ok. */ + error?: string; ciphertextBytes: number; plaintextBytes: number; /** Application key ids the object's inner ciphertext needs, each proven to open. */ @@ -70,17 +72,108 @@ export interface RestoreDrillReport { }; } +export interface RestoreDrillReport { + dateKey: string; + ageDays: number; + stale: boolean; + /** Every object of the newest date, in key order. */ + accounts: RestoreDrillAccount[]; + /** The accounts whose object did not pass. */ + failed: RestoreDrillAccount[]; +} + function countArray(value: unknown): number { return Array.isArray(value) ? value.length : 0; } +const EMPTY_COUNTS: RestoreDrillAccount["recordCounts"] = { + measurements: 0, + medications: 0, + intakeEvents: 0, + moodEntries: 0, +}; + /** - * Fetch → decrypt → parse the most recent off-host backup object. + * Fetch → decrypt → parse → key-check one object. Throws on every failure + * mode: fetch error, bad envelope / wrong key, malformed JSON, payload + * missing its core fields, inner ciphertext this server cannot open. + */ +async function drillObject( + s3: S3Like, + cfg: NonNullable>, + objectKey: string, + account: RestoreDrillAccount, +): Promise { + const ciphertext = await s3.getObject(objectKey); + account.ciphertextBytes = ciphertext.length; + // Read as a stream, never as one string: the JSON of a large record is + // longer than any string V8 can hold (#1031). The bulk tables are counted, + // not kept. + const source = openBackupObject(ciphertext, offhostKeyRing(cfg), objectKey); + async function* counted() { + for await (const chunk of source()) { + account.plaintextBytes += chunk.byteLength; + yield chunk; + } + } + // The envelope opening proves the off-host key. It proves nothing about + // the application keys the ciphertext INSIDE was written under, and that + // is the half a key rotation breaks: a restore of this object writes those + // values back verbatim. So every inner key id is collected, and values + // under each key are actually decrypted. + const keys = new BackupKeyIdCollector(); + let scanned; + try { + scanned = await scanBackupJson(counted(), { + streamKeys: new Set(["measurements", "intakeEvents", "moodEntries"]), + onElement: (key, element) => keys.visit(element, key), + }); + } catch (err) { + if (err instanceof BackupJsonError) { + throw new Error(`decrypted but is not a JSON object: ${err.message}`); + } + throw err; + } + const payload = scanned.document; + const streamedCounts = scanned.streamedCounts; + if ( + typeof payload.exportedAt !== "string" || + typeof payload.userId !== "string" || + streamedCounts.measurements === undefined + ) { + throw new Error( + "parses but is missing core fields (exportedAt / userId / measurements).", + ); + } + account.recordCounts = { + measurements: streamedCounts.measurements, + medications: countArray(payload.medications), + intakeEvents: streamedCounts.intakeEvents ?? 0, + moodEntries: streamedCounts.moodEntries ?? 0, + }; + + keys.visit(payload); + const keyVerdict = assessBackupKeys(keys, { + ignoreSections: new Set(["appSettings"]), + }); + account.innerKeyIds = keyVerdict.keyIds; + const keyProblem = describeBackupKeyProblem(keyVerdict); + if (keyProblem) { + throw new Error( + `opens, but its content could not be restored on this server. ${keyProblem}`, + ); + } +} + +/** + * Fetch → decrypt → parse every off-host backup object of the newest date, + * one account at a time. * * Read-only against the bucket (GetObject + ListObjects — both inside - * the uploader's existing IAM grant). Throws on every failure mode: - * not configured, empty bucket, fetch error, bad envelope / wrong key, - * malformed JSON, payload missing its core fields. + * the uploader's existing IAM grant). Throws when there is nothing to check + * (not configured, empty bucket); a failure of one account's object is + * recorded against that account and the others are still checked, so one + * account cannot turn the drill red without being named, nor hide another. */ export async function runRestoreDrill( s3Override?: S3Like, @@ -95,8 +188,8 @@ export async function runRestoreDrill( const s3 = s3Override ?? (await getS3Client(cfg)); // Date-prefixed keys (`YYYY-MM-DD/user-.json.enc`) sort - // lexicographically in chronological order, so the newest object is - // simply the maximum matching key. `_healthcheck/` probes and any + // lexicographically in chronological order, so the newest date is the + // date of the maximum matching key. `_healthcheck/` probes and any // foreign objects in the bucket are filtered out by the pattern. const objects = await s3.listObjects(""); const backupKeys = objects @@ -108,62 +201,30 @@ export async function runRestoreDrill( `Restore drill found no backup objects in bucket "${cfg.bucket}" — the nightly off-host upload is not producing artefacts.`, ); } - const objectKey = backupKeys[backupKeys.length - 1]; - const dateKey = BACKUP_KEY_PATTERN.exec(objectKey)![1]; + const dateKey = BACKUP_KEY_PATTERN.exec( + backupKeys[backupKeys.length - 1], + )![1]; + const newest = backupKeys.filter((k) => k.startsWith(`${dateKey}/`)); - const ciphertext = await s3.getObject(objectKey); - // Read as a stream, never as one string: the JSON of a large record is - // longer than any string V8 can hold (#1031). The bulk tables are counted, - // not kept. - const source = openBackupObject(ciphertext, offhostKeyRing(cfg), objectKey); - let plaintextBytes = 0; - async function* counted() { - for await (const chunk of source()) { - plaintextBytes += chunk.byteLength; - yield chunk; + const accounts: RestoreDrillAccount[] = []; + for (const objectKey of newest) { + const account: RestoreDrillAccount = { + objectKey, + ok: true, + ciphertextBytes: 0, + plaintextBytes: 0, + innerKeyIds: [], + recordCounts: { ...EMPTY_COUNTS }, + }; + try { + await drillObject(s3, cfg, objectKey, account); + } catch (err) { + account.ok = false; + account.error = `Restore drill: backup object "${objectKey}" ${ + err instanceof Error ? err.message : String(err) + }`; } - } - // The envelope opening proves the off-host key. It proves nothing about - // the application keys the ciphertext INSIDE was written under, and that - // is the half a key rotation breaks: a restore of this object writes those - // values back verbatim. So every inner key id is collected, and one value - // per key is actually decrypted. - const keys = new BackupKeyIdCollector(); - let scanned; - try { - scanned = await scanBackupJson(counted(), { - streamKeys: new Set(["measurements", "intakeEvents", "moodEntries"]), - onElement: (key, element) => keys.visit(element, key), - }); - } catch (err) { - if (err instanceof BackupJsonError) { - throw new Error( - `Restore drill: backup object "${objectKey}" decrypted but is not a JSON object: ${err.message}`, - ); - } - throw err; - } - const payload = scanned.document; - const streamedCounts = scanned.streamedCounts; - if ( - typeof payload.exportedAt !== "string" || - typeof payload.userId !== "string" || - streamedCounts.measurements === undefined - ) { - throw new Error( - `Restore drill: backup object "${objectKey}" parses but is missing core fields (exportedAt / userId / measurements).`, - ); - } - - keys.visit(payload); - const keyVerdict = assessBackupKeys(keys, { - ignoreSections: new Set(["appSettings"]), - }); - const keyProblem = describeBackupKeyProblem(keyVerdict); - if (keyProblem) { - throw new Error( - `Restore drill: backup object "${objectKey}" opens, but its content could not be restored on this server. ${keyProblem}`, - ); + accounts.push(account); } const ageDays = Math.floor( @@ -173,19 +234,11 @@ export async function runRestoreDrill( ); return { - objectKey, dateKey, ageDays, stale: ageDays > MAX_BACKUP_AGE_DAYS, - ciphertextBytes: ciphertext.length, - plaintextBytes, - innerKeyIds: keyVerdict.keyIds, - recordCounts: { - measurements: streamedCounts.measurements, - medications: countArray(payload.medications), - intakeEvents: streamedCounts.intakeEvents ?? 0, - moodEntries: streamedCounts.moodEntries ?? 0, - }, + accounts, + failed: accounts.filter((a) => !a.ok), }; } @@ -203,48 +256,65 @@ export async function handleRestoreDrill( return withBackgroundEvent("job.restore_drill", async (evt) => { try { const report = await runRestoreDrill(); - evt.addMeta("restore_drill_object_key", report.objectKey); - evt.addMeta("restore_drill_age_days", report.ageDays); - evt.addMeta("restore_drill_ciphertext_bytes", report.ciphertextBytes); - evt.addMeta("restore_drill_plaintext_bytes", report.plaintextBytes); + const total = (pick: (a: RestoreDrillAccount) => number) => + report.accounts.reduce((sum, a) => sum + pick(a), 0); + const meta = { + restore_drill_date: report.dateKey, + restore_drill_age_days: report.ageDays, + restore_drill_accounts: report.accounts.length, + restore_drill_accounts_failed: report.failed.length, + restore_drill_ciphertext_bytes: total((a) => a.ciphertextBytes), + restore_drill_plaintext_bytes: total((a) => a.plaintextBytes), + restore_drill_measurements: total((a) => a.recordCounts.measurements), + restore_drill_medications: total((a) => a.recordCounts.medications), + restore_drill_intake_events: total((a) => a.recordCounts.intakeEvents), + restore_drill_mood_entries: total((a) => a.recordCounts.moodEntries), + restore_drill_stale: report.stale, + }; + for (const [key, value] of Object.entries(meta)) evt.addMeta(key, value); evt.addMeta( - "restore_drill_measurements", - report.recordCounts.measurements, + "restore_drill_inner_key_ids", + [...new Set(report.accounts.flatMap((a) => a.innerKeyIds))] + .sort() + .join(","), ); - evt.addMeta("restore_drill_medications", report.recordCounts.medications); - evt.addMeta( - "restore_drill_intake_events", - report.recordCounts.intakeEvents, - ); - evt.addMeta( - "restore_drill_mood_entries", - report.recordCounts.moodEntries, - ); - evt.addMeta("restore_drill_stale", report.stale); - evt.addMeta("restore_drill_inner_key_ids", report.innerKeyIds.join(",")); + // One page per account that did not pass, naming its object, so the + // operator sees which account it is and that the others passed. + for (const account of report.failed) { + evt.addWarning(account.error ?? `${account.objectKey} failed`); + await reportWorkerError( + RESTORE_DRILL_QUEUE, + new Error(account.error ?? `${account.objectKey} failed`), + { + objectKey: account.objectKey, + accountsChecked: report.accounts.length, + accountsFailed: report.failed.length, + }, + ); + } if (report.stale) { await reportWorkerError( RESTORE_DRILL_QUEUE, new Error( `Newest off-host backup is ${report.ageDays} days old (threshold ${MAX_BACKUP_AGE_DAYS}) — the nightly upload chain has stalled.`, ), - { objectKey: report.objectKey, ageDays: report.ageDays }, + { dateKey: report.dateKey, ageDays: report.ageDays }, + ); + } + if (report.failed.length > 0) { + // Already paged above, per account. A retry reads the same objects + // and cannot change the verdict before the next upload. + return jobFailed( + `restore drill: ${report.failed.length} of ${report.accounts.length} accounts failed`, + new Error(report.failed.map((a) => a.objectKey).join(", ")), + meta, ); } // A stale chain is a verdict about the uploader, not a failure of the - // drill: the artefact was fetched, decrypted and parsed, and the page + // drill: the artefacts were fetched, decrypted and parsed, and the page // above already reached the operator. Failing the job here would retry a // reading that cannot change until the next upload lands. - return jobDone({ - restore_drill_age_days: report.ageDays, - restore_drill_ciphertext_bytes: report.ciphertextBytes, - restore_drill_plaintext_bytes: report.plaintextBytes, - restore_drill_measurements: report.recordCounts.measurements, - restore_drill_medications: report.recordCounts.medications, - restore_drill_intake_events: report.recordCounts.intakeEvents, - restore_drill_mood_entries: report.recordCounts.moodEntries, - restore_drill_stale: report.stale, - }); + return jobDone(meta); } catch (err) { if (err instanceof OffhostBackupNotConfiguredError) { evt.addWarning(`restore-drill skipped: ${err.message}`);