diff --git a/.env.example b/.env.example index aa5a7e3c3..3e8940f7f 100644 --- a/.env.example +++ b/.env.example @@ -233,6 +233,7 @@ API_TOKEN_HMAC_KEY="REPLACE-WITH-OUTPUT-OF-openssl-rand-hex-32" # BACKUP_S3_REGION="auto" # "auto" for Cloudflare R2; real region for AWS S3 # BACKUP_RETENTION_DAYS="30" # the number for your bucket lifecycle rule (see above) # BACKUP_ENCRYPTION_KEY="" # 64-hex-char or base64 32-byte; SEPARATE from ENCRYPTION_KEY +# BACKUP_ENCRYPTION_PREVIOUS_KEYS="" # retired off-host keys, comma-separated, read-only during a rotation # ----------------------------------------------------------------------------- # Stored backups in the database (optional) diff --git a/.env.production.example b/.env.production.example index ead149065..6f39567eb 100644 --- a/.env.production.example +++ b/.env.production.example @@ -345,13 +345,17 @@ API_TOKEN_HMAC_KEY="" # BACKUP_S3_REGION="auto" # # How long backup objects are meant to live, in days. Default 30, minimum 1. -# The worker never deletes a BACKUP object -- its grant is PutObject, GetObject -# and AbortMultipartUpload -- so this number is what you set the bucket's own -# lifecycle rule to. The one key it does delete is its own connection test's -# `_healthcheck/.bin`, and that delete is best-effort: without DeleteObject -# on that prefix every Test-connection press leaves one 1-byte orphan behind, -# silently. See docs/ops/backup-restore.md. +# The app does not expire objects by age: this number is what you set the +# bucket's own lifecycle rule to. It does delete objects in two cases, so the +# credential needs DeleteObject: every copy of an account that is deleted or +# whose data is wiped, and its own connection test's `_healthcheck/.bin`. +# See docs/ops/backup-restore.md. # BACKUP_RETENTION_DAYS="30" +# +# Off-host keys no longer written with, comma-separated, kept so older +# objects stay readable during a rotation of BACKUP_ENCRYPTION_KEY. Remove a +# key once the lifecycle rule has retired every object written under it. +# BACKUP_ENCRYPTION_PREVIOUS_KEYS="" # ----------------------------------------------------------------------------- # Stored backups in the database (optional) diff --git a/docker-compose.yml b/docker-compose.yml index a24b90ba0..7ad37e6da 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -125,6 +125,7 @@ x-healthlog-env: &healthlog_env # web = serve HTTP only; pair with a separate app-worker service HEALTHLOG_PROCESS_TYPE: "${HEALTHLOG_PROCESS_TYPE:-all}" BACKUP_ENCRYPTION_KEY: "${BACKUP_ENCRYPTION_KEY:-}" + BACKUP_ENCRYPTION_PREVIOUS_KEYS: "${BACKUP_ENCRYPTION_PREVIOUS_KEYS:-}" BACKUP_S3_ENDPOINT: "${BACKUP_S3_ENDPOINT:-}" BACKUP_S3_BUCKET: "${BACKUP_S3_BUCKET:-}" BACKUP_S3_ACCESS_KEY: "${BACKUP_S3_ACCESS_KEY:-}" diff --git a/scripts/env-manifest.json b/scripts/env-manifest.json index 11fcfadb5..6cd7c44e3 100644 --- a/scripts/env-manifest.json +++ b/scripts/env-manifest.json @@ -293,6 +293,11 @@ "purpose": "Trusted reverse-proxy hop count; the client IP is read that many entries from the right of X-Forwarded-For.", "example": "1" }, + { + "name": "BACKUP_ENCRYPTION_PREVIOUS_KEYS", + "purpose": "Comma-separated off-host backup keys that are no longer written with but may still be needed to read older objects. Set during a rotation of BACKUP_ENCRYPTION_KEY until the bucket's lifecycle rule has retired every object written under the old key.", + "example": "" + }, { "name": "BACKUP_MAX_STORED_MB", "purpose": "Largest stored backup of one account the app keeps, in megabytes of encrypted copy (2048 by default). A storage limit, independent of the app's memory: a backup past it fails with a message naming this variable, and the previous copy stays.", diff --git a/scripts/restore-backup.ts b/scripts/restore-backup.ts index c6f4a51fe..15cc8e80a 100644 --- a/scripts/restore-backup.ts +++ b/scripts/restore-backup.ts @@ -32,6 +32,7 @@ import { loadOffhostConfig, getS3Client, openBackupObject, + offhostKeyRing, OffhostBackupNotConfiguredError, } from "@/lib/jobs/offhost-backup"; @@ -109,7 +110,7 @@ async function main() { `Downloading s3://${cfg.bucket}/${key} from ${cfg.endpoint} (region=${cfg.region})`, ); const ciphertext = await s3.getObject(key); - const source = openBackupObject(ciphertext, cfg.encryptionKey); + const source = openBackupObject(ciphertext, offhostKeyRing(cfg), key); // Checked first, in one streamed pass that keeps only the small sections, // so nothing is written for an object that is not a backup. diff --git a/src/app/api/admin/backups/upload/route.ts b/src/app/api/admin/backups/upload/route.ts index 6e6ba6e4a..907402a78 100644 --- a/src/app/api/admin/backups/upload/route.ts +++ b/src/app/api/admin/backups/upload/route.ts @@ -29,7 +29,6 @@ * Phase B1 / criterion 2 of the v1.4.15 backup-completeness work. */ import { Readable } from "node:stream"; -import { createGunzip } from "node:zlib"; import { NextRequest } from "next/server"; import { ZodError } from "zod/v4"; @@ -38,6 +37,10 @@ import { apiHandler, HttpError, requireAdmin } from "@/lib/api-handler"; import { apiError, apiSuccess, getClientIp } from "@/lib/api-response"; import { auditLog } from "@/lib/auth/audit"; import { BackupJsonError, scanBackupJson } from "@/lib/export/backup-json-scan"; +import { + BackupUploadDecodeError, + decodeBackupUpload, +} from "@/lib/export/backup-upload-decode"; import { assessBackupKeys, BACKUP_KEY_MISSING_CODE, @@ -99,50 +102,22 @@ class UploadRefused extends Error { } } -/** Bytes → bytes, gunzipped when the file starts with the gzip magic. */ +/** The file's bytes, gunzipped and counted on both ends. */ async function* decodedBytes( source: AsyncIterable, limit: number, ): AsyncGenerator { - let seen = 0; - async function* counted() { - for await (const chunk of source) { - seen += chunk.byteLength; - if (seen > limit) { - throw new UploadRefused( - 413, - `Upload exceeds ${Math.round(limit / 1024 / 1024)} MB limit`, - { reason: "file_size_exceeded", size: seen }, - ); - } - yield chunk; - } - } - const iterator = counted()[Symbol.asyncIterator](); - const first = await iterator.next(); - if (first.done) return; - const rest = (async function* () { - yield first.value; - for (;;) { - const next = await iterator.next(); - if (next.done) return; - yield next.value; - } - })(); - const head = first.value; - if (head.byteLength >= 2 && head[0] === 0x1f && head[1] === 0x8b) { - const gunzip = Readable.from(rest).pipe(createGunzip()); - try { - for await (const chunk of gunzip) yield chunk as Buffer; - } catch (err) { - if (err instanceof UploadRefused) throw err; - throw new UploadRefused(422, "Uploaded file is not valid gzip", { - reason: "invalid_gzip", + try { + yield* decodeBackupUpload(source, { compressedLimit: limit }); + } catch (err) { + if (err instanceof BackupUploadDecodeError) { + throw new UploadRefused(err.status, err.message, { + reason: err.reason, + ...(err.size !== undefined ? { size: err.size } : {}), }); } - return; + throw err; } - yield* rest; } export const POST = apiHandler(async (request: NextRequest) => { diff --git a/src/lib/crypto.ts b/src/lib/crypto.ts index 6493fee3c..972b8d21a 100644 --- a/src/lib/crypto.ts +++ b/src/lib/crypto.ts @@ -464,13 +464,21 @@ export interface RawStreamEncryptor { final(): Buffer; } -/** Open a byte-level streaming encryptor under an explicit 32-byte key. */ -export function createRawStreamEncryptor(key: Buffer): RawStreamEncryptor { +/** + * Open a byte-level streaming encryptor under an explicit 32-byte key. + * `aad`, when given, is authenticated with the ciphertext and has to be given + * again, byte for byte, to decrypt it. + */ +export function createRawStreamEncryptor( + key: Buffer, + aad?: Buffer, +): RawStreamEncryptor { if (key.byteLength !== 32) { throw new Error("Stream encryption key must be 32 bytes"); } const iv = randomBytes(IV_LENGTH); const cipher = createCipheriv(ALGORITHM, key, iv); + if (aad) cipher.setAAD(aad); let done = false; return { @@ -498,7 +506,11 @@ export function createRawStreamEncryptor(key: Buffer): RawStreamEncryptor { * unverified plaintext to save a copy would trade the authentication for * memory. */ -export function decryptRawStream(packed: Buffer, key: Buffer): Buffer { +export function decryptRawStream( + packed: Buffer, + key: Buffer, + aad?: Buffer, +): Buffer { if (packed.byteLength < IV_LENGTH + AUTH_TAG_LENGTH) { throw new Error("Streamed ciphertext is truncated"); } @@ -507,6 +519,7 @@ export function decryptRawStream(packed: Buffer, key: Buffer): Buffer { const ct = packed.subarray(IV_LENGTH, packed.byteLength - AUTH_TAG_LENGTH); const dec = createDecipheriv(ALGORITHM, key, iv); dec.setAuthTag(tag); + if (aad) dec.setAAD(aad); return Buffer.concat([dec.update(ct), dec.final()]); } diff --git a/src/lib/export/__tests__/backup-upload-decode.test.ts b/src/lib/export/__tests__/backup-upload-decode.test.ts new file mode 100644 index 000000000..ec41aae02 --- /dev/null +++ b/src/lib/export/__tests__/backup-upload-decode.test.ts @@ -0,0 +1,70 @@ +/** + * An uploaded backup is counted after the gunzip as well as before it. + * Mutation that must turn this red: drop the inflated count (the bomb reads + * through to the end). + */ +import { gzipSync } from "node:zlib"; + +import { describe, expect, it } from "vitest"; + +import { + BackupUploadDecodeError, + decodeBackupUpload, +} from "../backup-upload-decode"; + +async function drain( + bytes: Buffer, + limits: Parameters[1], +): Promise { + async function* source() { + for (let i = 0; i < bytes.length; i += 4096) { + yield bytes.subarray(i, i + 4096); + } + } + const out: Buffer[] = []; + for await (const chunk of decodeBackupUpload(source(), limits)) { + out.push(Buffer.from(chunk)); + } + return Buffer.concat(out); +} + +describe("decodeBackupUpload", () => { + it("passes plain JSON through and gunzips a gzip file", async () => { + const json = Buffer.from(JSON.stringify({ userId: "u1", n: [1, 2, 3] })); + expect(await drain(json, { compressedLimit: 1024 })).toEqual(json); + expect(await drain(gzipSync(json), { compressedLimit: 1024 })).toEqual( + json, + ); + }); + + it("refuses a gzip that inflates far past its own size", async () => { + const bomb = gzipSync(Buffer.alloc(32 * 1024 * 1024)); + expect(bomb.length).toBeLessThan(64 * 1024); + const refused = drain(bomb, { + compressedLimit: 1024 * 1024, + minAllowance: 1024 * 1024, + ratio: 100, + }); + await expect(refused).rejects.toBeInstanceOf(BackupUploadDecodeError); + await expect(refused).rejects.toMatchObject({ + status: 413, + reason: "inflated_size_exceeded", + }); + }); + + it("refuses a file that is not valid gzip", async () => { + const broken = Buffer.concat([ + Buffer.from([0x1f, 0x8b]), + Buffer.from("not gzip"), + ]); + await expect( + drain(broken, { compressedLimit: 1024 }), + ).rejects.toMatchObject({ status: 422, reason: "invalid_gzip" }); + }); + + it("refuses more bytes than the upload limit", async () => { + await expect( + drain(Buffer.alloc(10_000, 0x20), { compressedLimit: 5_000 }), + ).rejects.toMatchObject({ status: 413, reason: "file_size_exceeded" }); + }); +}); diff --git a/src/lib/export/backup-upload-decode.ts b/src/lib/export/backup-upload-decode.ts new file mode 100644 index 000000000..4db4dc3dd --- /dev/null +++ b/src/lib/export/backup-upload-decode.ts @@ -0,0 +1,116 @@ +/** + * An uploaded backup file's bytes, gunzipped when they are gzip, with both + * ends counted. + * + * The upload counted what arrived and nothing after the gunzip, so a few + * megabytes of gzip could inflate without limit. The JSON scanner keeps every + * section but the measurements in memory, which turned such a file into an + * out-of-memory kill of the whole app. The inflated bytes are now held to + * {@link INFLATION_RATIO} times the compressed bytes read so far (a real + * backup compresses about tenfold), never less than + * {@link MIN_INFLATED_ALLOWANCE}, and never more than {@link MAX_INFLATED_BYTES}. + */ +import { Readable } from "node:stream"; +import { createGunzip } from "node:zlib"; + +/** How far an uploaded gzip may expand, measured on the bytes read so far. */ +export const INFLATION_RATIO = 100; +/** Below this much output the ratio is not what protects the process. */ +export const MIN_INFLATED_ALLOWANCE = 256 * 1024 * 1024; +/** + * The largest file an upload may inflate to. A disaster-recovery backup of + * 1.25 million readings is 662 MB of JSON; this leaves room for several times + * that record and stops a bomb long before memory does. + */ +export const MAX_INFLATED_BYTES = 4 * 1024 * 1024 * 1024; + +/** Thrown for a file that is too large or not valid gzip. */ +export class BackupUploadDecodeError extends Error { + constructor( + readonly status: 413 | 422, + message: string, + readonly reason: + "file_size_exceeded" | "inflated_size_exceeded" | "invalid_gzip", + readonly size?: number, + ) { + super(message); + this.name = "BackupUploadDecodeError"; + } +} + +export interface DecodeLimits { + /** Cap on the bytes as they arrive. */ + compressedLimit: number; + ratio?: number; + minAllowance?: number; + maxInflated?: number; +} + +/** Bytes → bytes, gunzipped when the file starts with the gzip magic. */ +export async function* decodeBackupUpload( + source: AsyncIterable, + limits: DecodeLimits, +): AsyncGenerator { + const ratio = limits.ratio ?? INFLATION_RATIO; + const minAllowance = limits.minAllowance ?? MIN_INFLATED_ALLOWANCE; + const maxInflated = limits.maxInflated ?? MAX_INFLATED_BYTES; + let seen = 0; + async function* counted() { + for await (const chunk of source) { + seen += chunk.byteLength; + if (seen > limits.compressedLimit) { + throw new BackupUploadDecodeError( + 413, + `Upload exceeds ${Math.round(limits.compressedLimit / 1024 / 1024)} MB limit`, + "file_size_exceeded", + seen, + ); + } + yield chunk; + } + } + const iterator = counted()[Symbol.asyncIterator](); + const first = await iterator.next(); + if (first.done) return; + const rest = (async function* () { + yield first.value; + for (;;) { + const next = await iterator.next(); + if (next.done) return; + yield next.value; + } + })(); + const head = first.value; + if (!(head.byteLength >= 2 && head[0] === 0x1f && head[1] === 0x8b)) { + yield* rest; + return; + } + const gunzip = Readable.from(rest).pipe(createGunzip()); + let inflated = 0; + try { + for await (const chunk of gunzip) { + inflated += (chunk as Buffer).byteLength; + const allowed = Math.min( + maxInflated, + Math.max(minAllowance, seen * ratio), + ); + if (inflated > allowed) { + gunzip.destroy(); + throw new BackupUploadDecodeError( + 413, + `The uploaded file expands to more than ${Math.round(allowed / 1024 / 1024)} MB, which no backup this server wrote does. Nothing was stored.`, + "inflated_size_exceeded", + inflated, + ); + } + yield chunk as Buffer; + } + } catch (err) { + if (err instanceof BackupUploadDecodeError) throw err; + throw new BackupUploadDecodeError( + 422, + "Uploaded file is not valid gzip", + "invalid_gzip", + ); + } +} diff --git a/src/lib/jobs/__tests__/offhost-backup.test.ts b/src/lib/jobs/__tests__/offhost-backup.test.ts index 57c9c07bc..d038b3e73 100644 --- a/src/lib/jobs/__tests__/offhost-backup.test.ts +++ b/src/lib/jobs/__tests__/offhost-backup.test.ts @@ -224,7 +224,11 @@ describe("runOffhostBackup", () => { expect(s3.deleteObject).not.toHaveBeenCalled(); const ct = s3.store.get("2026-05-08/user-u1.json.enc")!; - const decoded = decryptBackup(ct, Buffer.from(ENC_KEY, "hex")); + const decoded = decryptBackup( + ct, + Buffer.from(ENC_KEY, "hex"), + "2026-05-08/user-u1.json.enc", + ); const parsed = JSON.parse(decoded); expect(parsed.userId).toBe("u1"); expect(() => backupPayloadSchema.parse(parsed)).not.toThrow(); @@ -343,7 +347,13 @@ describe("runOffhostBackup", () => { expect(report.uploaded).toBe(1); const ciphertext = s3.store.get("2026-05-08/user-u1.json.enc")!; expect( - JSON.parse(decryptBackup(ciphertext, Buffer.from(ENC_KEY, "hex"))), + JSON.parse( + decryptBackup( + ciphertext, + Buffer.from(ENC_KEY, "hex"), + "2026-05-08/user-u1.json.enc", + ), + ), ).toEqual(canonicalPayload); expect(() => backupPayloadSchema.parse(canonicalPayload)).not.toThrow(); }); @@ -509,8 +519,8 @@ describe("uploadEncryptedBackup", () => { const stored = s3.store.get("k")!; expect(stored.byteLength).toBe(bytes); - expect(stored.subarray(0, 5).toString("binary")).toBe("HLBK\x03"); - expect(decryptBackup(stored, key)).toBe(document); + expect(stored.subarray(0, 5).toString("binary")).toBe("HLBK\x04"); + expect(decryptBackup(stored, key, "k")).toBe(document); }); it("restores an object of the old shape and one of the new one alike", async () => { @@ -526,10 +536,53 @@ describe("uploadEncryptedBackup", () => { const oldBytes = s3.store.get("old")!; const newBytes = s3.store.get("new")!; expect(oldBytes.subarray(0, 5).toString("binary")).toBe("HLBK\x02"); - expect(newBytes.subarray(0, 5).toString("binary")).toBe("HLBK\x03"); + expect(newBytes.subarray(0, 5).toString("binary")).toBe("HLBK\x04"); // Different framing, same record, one reader. expect(decryptBackup(oldBytes, key)).toBe(document); - expect(decryptBackup(newBytes, key)).toBe(document); + expect(decryptBackup(newBytes, key, "new")).toBe(document); + }); + + it("binds a new object to its key in the bucket", async () => { + const s3 = makeS3Mock(); + const document = JSON.stringify({ userId: "u1" }); + await uploadEncryptedBackup( + s3, + "2026-09-01/user-u1.json.enc", + key, + (write) => write(document), + ); + const stored = s3.store.get("2026-09-01/user-u1.json.enc")!; + expect(decryptBackup(stored, key, "2026-09-01/user-u1.json.enc")).toBe( + document, + ); + // The same bytes presented as another account's copy, or another night's, + // do not open. + expect(() => + decryptBackup(stored, key, "2026-09-01/user-u2.json.enc"), + ).toThrow(); + expect(() => + decryptBackup(stored, key, "2026-09-02/user-u1.json.enc"), + ).toThrow(); + }); + + it("reads objects under a retired key through the previous-keys ring", async () => { + const s3 = makeS3Mock(); + const document = JSON.stringify({ userId: "u1" }); + const oldKey = Buffer.alloc(32, 7); + const newKey = Buffer.alloc(32, 9); + await uploadEncryptedBackup(s3, "k4", oldKey, (write) => write(document)); + await s3.putObject("k2", encryptBackup(document, oldKey)); + const ring = { active: newKey, previous: [oldKey] }; + expect(decryptBackup(s3.store.get("k4")!, ring, "k4")).toBe(document); + expect(decryptBackup(s3.store.get("k2")!, ring)).toBe(document); + // Without the retired key the new-format object names the key it needs. + expect(() => + decryptBackup( + s3.store.get("k4")!, + { active: newKey, previous: [] }, + "k4", + ), + ).toThrow(/BACKUP_ENCRYPTION_PREVIOUS_KEYS/); }); it("rejects a tampered version-3 object rather than returning a partial one", async () => { diff --git a/src/lib/jobs/offhost-backup.ts b/src/lib/jobs/offhost-backup.ts index 9a21b15f2..73f01f0f2 100644 --- a/src/lib/jobs/offhost-backup.ts +++ b/src/lib/jobs/offhost-backup.ts @@ -22,7 +22,12 @@ * a finished object. See docs/ops/backup-restore.md. */ import { Buffer } from "node:buffer"; -import { createCipheriv, createDecipheriv, randomBytes } from "node:crypto"; +import { + createCipheriv, + createDecipheriv, + createHash, + randomBytes, +} from "node:crypto"; import { Readable, Transform } from "node:stream"; import { createGunzip, createGzip, gunzipSync, gzipSync } from "node:zlib"; import type { PrismaClient } from "@/generated/prisma/client"; @@ -41,7 +46,16 @@ export interface OffhostBackupConfig { accessKey: string; secretKey: string; region: string; + /** The key new objects are written under (`BACKUP_ENCRYPTION_KEY`). */ encryptionKey: Buffer; + /** + * Keys older objects may still be under (`BACKUP_ENCRYPTION_PREVIOUS_KEYS`, + * comma-separated). Read only; nothing is written under them. This is the + * rotation path: the new key goes into `BACKUP_ENCRYPTION_KEY`, the old one + * here, until the bucket's lifecycle rule has retired every object written + * under it. + */ + previousEncryptionKeys: Buffer[]; } export class OffhostBackupNotConfiguredError extends Error { @@ -105,6 +119,11 @@ export function loadOffhostConfig(): OffhostBackupConfig | null { secretKey, region: process.env.BACKUP_S3_REGION ?? "auto", encryptionKey: decodeBackupKey(encRaw), + previousEncryptionKeys: (process.env.BACKUP_ENCRYPTION_PREVIOUS_KEYS ?? "") + .split(",") + .map((raw) => raw.trim()) + .filter((raw) => raw.length > 0) + .map(decodeBackupKey), }; } @@ -115,6 +134,9 @@ export function loadOffhostConfig(): OffhostBackupConfig | null { * 1: magic(4)="HLBK" || 0x01 || iv(12) || tag(16) || ciphertext(json) * 2: magic(4)="HLBK" || 0x02 || iv(12) || tag(16) || ciphertext(gzip(json)) * 3: magic(4)="HLBK" || 0x03 || iv(12) || ciphertext(gzip(json)) || tag(16) + * 4: magic(4)="HLBK" || 0x04 || keyIdLen(1) || keyId || iv(12) + * || ciphertext(gzip(json)) || tag(16), with the object's own key + * (`/user-.json.enc`) as GCM associated data * * Version 1 encrypted the JSON directly; version 2 gzipped it first. Both put * the tag in front of the ciphertext, and that is precisely what could not be @@ -135,10 +157,108 @@ export function loadOffhostConfig(): OffhostBackupConfig | null { const BACKUP_ENVELOPE_PLAIN = 0x01; const BACKUP_ENVELOPE_GZIP = 0x02; const BACKUP_ENVELOPE_STREAM = 0x03; +/** + * Version 4 adds two things version 3 did not have, and changes nothing else. + * + * A key id: the first twelve hex characters of the key's SHA-256, so a reader + * holding several keys (`BACKUP_ENCRYPTION_PREVIOUS_KEYS` during a rotation) + * knows which one opens an object instead of trying each. It is a + * fingerprint, not a name the operator has to keep in step. + * + * Associated data: the object's key in the bucket, which names the account + * and the night. Before this, an object copied to another account's key in + * the bucket, or to another date, decrypted as if it belonged there; now it + * does not open at all. + */ +const BACKUP_ENVELOPE_KEYED = 0x04; +const OFFHOST_AAD_PREFIX = "healthlog/offhost-backup/v4|"; const MAGIC = "HLBK"; /** magic(4) + version(1). Where the per-version body begins. */ const PREAMBLE_LENGTH = 5; +/** The off-host keys a reader may use: the current one, then the retired. */ +export interface OffhostKeyRing { + active: Buffer; + previous: Buffer[]; +} + +/** The key ring of a loaded configuration. */ +export function offhostKeyRing(cfg: OffhostBackupConfig): OffhostKeyRing { + return { active: cfg.encryptionKey, previous: cfg.previousEncryptionKeys }; +} + +function asKeyRing(key: Buffer | OffhostKeyRing): OffhostKeyRing { + return Buffer.isBuffer(key) ? { active: key, previous: [] } : key; +} + +/** The id a version-4 object records for the key it was written under. */ +export function offhostKeyId(key: Buffer): string { + return createHash("sha256").update(key).digest("hex").slice(0, 12); +} + +function offhostAad(objectKey: string): Buffer { + return Buffer.from(`${OFFHOST_AAD_PREFIX}${objectKey}`, "utf8"); +} + +/** + * Open the gzip bytes of a version-4 object: find its key by id, verify the + * tag with the object key as associated data. + */ +function openKeyed( + buf: Buffer, + ring: OffhostKeyRing, + objectKey: string | undefined, +): Buffer { + if (objectKey === undefined) { + throw new Error( + "This backup object is bound to its key in the bucket; pass the object key to open it", + ); + } + const idLen = buf[PREAMBLE_LENGTH]; + const keyId = buf + .subarray(PREAMBLE_LENGTH + 1, PREAMBLE_LENGTH + 1 + idLen) + .toString("latin1"); + const key = [ring.active, ...ring.previous].find( + (candidate) => offhostKeyId(candidate) === keyId, + ); + if (!key) { + throw new Error( + `Backup object was written under off-host key ${keyId}, which is neither BACKUP_ENCRYPTION_KEY nor in BACKUP_ENCRYPTION_PREVIOUS_KEYS`, + ); + } + return decryptRawStream( + buf.subarray(PREAMBLE_LENGTH + 1 + idLen), + key, + offhostAad(objectKey), + ); +} + +/** + * Try each key of the ring on an object with no key id (versions 1 to 3), + * current first. GCM refuses a wrong key outright, so the first key that + * opens it is the one it was written under. + */ +function withEachKey(ring: OffhostKeyRing, open: (key: Buffer) => T): T { + let lastError: unknown = null; + for (const key of [ring.active, ...ring.previous]) { + try { + return open(key); + } catch (err) { + lastError = err; + } + } + throw lastError ?? new Error("No off-host key configured"); +} + +function isKnownVersion(version: number): boolean { + return ( + version === BACKUP_ENVELOPE_PLAIN || + version === BACKUP_ENVELOPE_GZIP || + version === BACKUP_ENVELOPE_STREAM || + version === BACKUP_ENVELOPE_KEYED + ); +} + /** * Write a whole JSON string as a version-2 object. * @@ -163,23 +283,36 @@ export function encryptBackup(plaintext: string, key: Buffer): Buffer { return Buffer.concat([header, iv, tag, ct]); } -export function decryptBackup(buf: Buffer, key: Buffer): string { +export function decryptBackup( + buf: Buffer, + keys: Buffer | OffhostKeyRing, + objectKey?: string, +): string { const magic = buf.subarray(0, 4).toString("binary"); const version = buf[PREAMBLE_LENGTH - 1]; - if ( - magic !== MAGIC || - (version !== BACKUP_ENVELOPE_PLAIN && - version !== BACKUP_ENVELOPE_GZIP && - version !== BACKUP_ENVELOPE_STREAM) - ) { + if (magic !== MAGIC || !isKnownVersion(version)) { throw new Error("Invalid backup envelope (bad magic or version)"); } + const ring = asKeyRing(keys); + if (version === BACKUP_ENVELOPE_KEYED) { + return gunzipSync(openKeyed(buf, ring, objectKey)).toString("utf8"); + } if (version === BACKUP_ENVELOPE_STREAM) { // iv | ciphertext | tag, exactly what the streaming writer emits and what // the shared reader verifies whole before it hands back a byte. - const plaintext = decryptRawStream(buf.subarray(PREAMBLE_LENGTH), key); + const plaintext = withEachKey(ring, (key) => + decryptRawStream(buf.subarray(PREAMBLE_LENGTH), key), + ); return gunzipSync(plaintext).toString("utf8"); } + const plaintext = withEachKey(ring, (key) => openLeadingTag(buf, key)); + return version === BACKUP_ENVELOPE_GZIP + ? gunzipSync(plaintext).toString("utf8") + : plaintext.toString("utf8"); +} + +/** Versions 1 and 2: iv | tag | ciphertext. */ +function openLeadingTag(buf: Buffer, key: Buffer): Buffer { const iv = buf.subarray(PREAMBLE_LENGTH, PREAMBLE_LENGTH + IV_LENGTH); const tag = buf.subarray( PREAMBLE_LENGTH + IV_LENGTH, @@ -188,10 +321,7 @@ export function decryptBackup(buf: Buffer, key: Buffer): string { const ct = buf.subarray(PREAMBLE_LENGTH + IV_LENGTH + TAG_LENGTH); const dec = createDecipheriv(ALGORITHM, key, iv); dec.setAuthTag(tag); - const plaintext = Buffer.concat([dec.update(ct), dec.final()]); - return version === BACKUP_ENVELOPE_GZIP - ? gunzipSync(plaintext).toString("utf8") - : plaintext.toString("utf8"); + return Buffer.concat([dec.update(ct), dec.final()]); } /** @@ -204,31 +334,24 @@ export function decryptBackup(buf: Buffer, key: Buffer): string { */ export function openBackupObject( buf: Buffer, - key: Buffer, + keys: Buffer | OffhostKeyRing, + objectKey?: string, ): () => AsyncIterable { const magic = buf.subarray(0, 4).toString("binary"); const version = buf[PREAMBLE_LENGTH - 1]; - if ( - magic !== MAGIC || - (version !== BACKUP_ENVELOPE_PLAIN && - version !== BACKUP_ENVELOPE_GZIP && - version !== BACKUP_ENVELOPE_STREAM) - ) { + if (magic !== MAGIC || !isKnownVersion(version)) { throw new Error("Invalid backup envelope (bad magic or version)"); } + const ring = asKeyRing(keys); let plaintext: Buffer; - if (version === BACKUP_ENVELOPE_STREAM) { - plaintext = decryptRawStream(buf.subarray(PREAMBLE_LENGTH), key); - } else { - const iv = buf.subarray(PREAMBLE_LENGTH, PREAMBLE_LENGTH + IV_LENGTH); - const tag = buf.subarray( - PREAMBLE_LENGTH + IV_LENGTH, - PREAMBLE_LENGTH + IV_LENGTH + TAG_LENGTH, + if (version === BACKUP_ENVELOPE_KEYED) { + plaintext = openKeyed(buf, ring, objectKey); + } else if (version === BACKUP_ENVELOPE_STREAM) { + plaintext = withEachKey(ring, (key) => + decryptRawStream(buf.subarray(PREAMBLE_LENGTH), key), ); - const ct = buf.subarray(PREAMBLE_LENGTH + IV_LENGTH + TAG_LENGTH); - const dec = createDecipheriv(ALGORITHM, key, iv); - dec.setAuthTag(tag); - plaintext = Buffer.concat([dec.update(ct), dec.final()]); + } else { + plaintext = withEachKey(ring, (key) => openLeadingTag(buf, key)); } if (version === BACKUP_ENVELOPE_PLAIN) { return () => Readable.from([plaintext]); @@ -306,12 +429,15 @@ export class OffhostBackupTooLargeError extends Error { */ function createEnvelopeStream( key: Buffer, + objectKey: string, limitBytes: number, ): { stream: Transform; bytes: () => number } { - const encryptor = createRawStreamEncryptor(key); + const encryptor = createRawStreamEncryptor(key, offhostAad(objectKey)); + const keyId = Buffer.from(offhostKeyId(key), "latin1"); const header = Buffer.concat([ Buffer.from(MAGIC, "binary"), - Buffer.from([BACKUP_ENVELOPE_STREAM]), + Buffer.from([BACKUP_ENVELOPE_KEYED, keyId.byteLength]), + keyId, encryptor.iv, ]); let written = 0; @@ -398,6 +524,7 @@ export async function uploadEncryptedBackup( const gzip = createGzip(); const { stream: envelope, bytes } = createEnvelopeStream( encryptionKey, + objectKey, limitBytes, ); diff --git a/src/lib/jobs/restore-drill.ts b/src/lib/jobs/restore-drill.ts index 1cfee7dff..4571af414 100644 --- a/src/lib/jobs/restore-drill.ts +++ b/src/lib/jobs/restore-drill.ts @@ -30,6 +30,7 @@ import { } from "@/lib/export/backup-key-ids"; import { openBackupObject, + offhostKeyRing, getS3Client, loadOffhostConfig, OffhostBackupNotConfiguredError, @@ -114,7 +115,7 @@ export async function runRestoreDrill( // Read as a stream, never as one string: the JSON of a large record is // longer than any string V8 can hold (#1031). The bulk tables are counted, // not kept. - const source = openBackupObject(ciphertext, cfg.encryptionKey); + const source = openBackupObject(ciphertext, offhostKeyRing(cfg), objectKey); let plaintextBytes = 0; async function* counted() { for await (const chunk of source()) { diff --git a/tests/integration/admin-backups-canonical-roundtrip.test.ts b/tests/integration/admin-backups-canonical-roundtrip.test.ts index 44516b544..f26ff943c 100644 --- a/tests/integration/admin-backups-canonical-roundtrip.test.ts +++ b/tests/integration/admin-backups-canonical-roundtrip.test.ts @@ -549,7 +549,11 @@ describe("canonical disaster-recovery backup round-trip", () => { const offhostObject = s3.objects.get(`2026-07-02/user-${ownerId}.json.enc`); expect(offhostObject).toBeDefined(); const payload = parseBackupPayload( - decryptBackup(offhostObject!, Buffer.from(OFFHOST_ENCRYPTION_KEY, "hex")), + decryptBackup( + offhostObject!, + Buffer.from(OFFHOST_ENCRYPTION_KEY, "hex"), + `2026-07-02/user-${ownerId}.json.enc`, + ), ); expect(payload).toMatchObject({ schemaVersion: "2", diff --git a/tests/integration/offhost-backup-streaming-memory.test.ts b/tests/integration/offhost-backup-streaming-memory.test.ts index c5193fd65..0c253dd69 100644 --- a/tests/integration/offhost-backup-streaming-memory.test.ts +++ b/tests/integration/offhost-backup-streaming-memory.test.ts @@ -273,7 +273,9 @@ describe("off-host backup under a memory budget", () => { ).toBeLessThan(STREAM_BUDGET_BYTES); // An object that writes but does not read is worse than none. - const restored = JSON.parse(decryptBackup(s3.kept!, BACKUP_KEY)) as { + const restored = JSON.parse( + decryptBackup(s3.kept!, BACKUP_KEY, objectKey), + ) as { measurements: Array<{ deletedAt: string | null }>; moodEntries: unknown[]; intakeEvents: unknown[];