/** * Which application keys the ciphertext INSIDE a backup was written under. * * A disaster-recovery backup carries every encrypted column the way the * database holds it: a note, a document, a coach message travel as the stored * ciphertext, base64-encoded, and a restore writes that ciphertext back * verbatim. The envelope around the whole copy is sealed separately, and key * rotation re-seals that envelope, but it cannot reach inside it, and it * cannot reach a copy in the off-host bucket or a file an operator keeps on a * disk at all. So a backup taken before a rotation still needs the key the * rotation retired, and a restore without that key writes rows back that no * reader can open. Nothing in the restore used to notice: the envelope opened, * the schema passed, and the damage showed up later as notes that would not * decrypt. * * This module answers the question the restore has to ask first: which key * ids does this file need, and are they all here. Every encrypted column in a * backup payload is named `…Encrypted`, and each value is one of: * * - the string codec as stored in a text column: `.`; * - a Bytes column, base64-encoded, holding the string codec as UTF-8; * - a Bytes column, base64-encoded, holding the binary codec * (`0x02 | keyIdLen | keyId | iv | tag | ct`); * - a legacy value with no key id, which only the `v1` key opens: in a * text column the bare base64 itself, in a Bytes column that base64 * again, encoded. * * Two readers share the classification: a walk over a parsed document (the * restore, the preview, the upload and the restore drill already hold one), * and a text scanner for a backup that is being written, which never exists * as a document (`createBackupKeyIdTextScanner`). */ import { Buffer } from "node:buffer"; import { decrypt, decryptBytes, getConfiguredKeyIds } from "@/lib/crypto"; import { ENCRYPTED_COLUMNS } from "@/lib/crypto/encrypted-columns"; /** The key id a value with no key id prefix was written under. */ export const LEGACY_INNER_KEY_ID = "v1"; const KEY_ID = /^[A-Za-z0-9_-]{1,32}$/; const VERSIONED = /^([A-Za-z0-9_-]{1,32})\.[A-Za-z0-9+/=]+$/; const BASE64 = /^[A-Za-z0-9+/]+={0,2}$/; const BASE64_TAIL = /^[A-Za-z0-9+/=]*$/; /** Every byte of a decoded head is a base64 character. */ const BASE64_TEXT = /^[A-Za-z0-9+/=]+$/; const BYTES_CODEC_VERSION = 0x02; const MIN_CIPHERTEXT_CHARS = 40; /** How an inner value is laid out, which decides how a sample is opened. */ export type InnerCiphertextForm = "string" | "bytes-string" | "binary"; export interface InnerCiphertext { keyId: string; form: InnerCiphertextForm; } /** * The key id one inner ciphertext value needs, or null for something that is * not ciphertext at all (an empty string). Decodes only a prefix, so a * multi-megabyte document costs the same as a note. */ export function innerCiphertextKeyId(value: string): InnerCiphertext | null { // The shortest ciphertext either codec writes is 28 bytes (nonce and tag), // 40 characters as base64: anything shorter is not ciphertext, and reading // it as a legacy value would demand a key for a field that holds none. if (value.length < MIN_CIPHERTEXT_CHARS) return null; const versioned = VERSIONED.exec(value.slice(0, 64)); if (versioned && value.indexOf(".") === versioned[1].length) { return { keyId: versioned[1], form: "string" }; } // A Bytes column, base64-encoded. 48 characters decode to 36 bytes, which // covers the longest header either codec can have (2 + 32 for the binary // one, 33 for `.`). const head = value.slice(0, 48); if (!BASE64.test(head)) return null; const bytes = Buffer.from(head, "base64"); if (bytes.length >= 3 && bytes[0] === BYTES_CODEC_VERSION) { const keyIdLen = bytes[1]; if (keyIdLen >= 1 && keyIdLen <= 32 && bytes.length >= 2 + keyIdLen) { const keyId = bytes.subarray(2, 2 + keyIdLen).toString("latin1"); if (KEY_ID.test(keyId)) return { keyId, form: "binary" }; } } const text = bytes.toString("latin1"); const dot = text.indexOf("."); // The rest of the prefix has to read as base64 too, or a legacy value's // random bytes could pass for `.` about once in a thousand values. if ( dot > 0 && KEY_ID.test(text.slice(0, dot)) && BASE64_TAIL.test(text.slice(dot + 1)) ) { return { keyId: text.slice(0, dot), form: "bytes-string" }; } // No key id anywhere: the legacy layout, which `decrypt` opens with `v1`. // Which column it came from decides how it opens. A Bytes column holding a // legacy value carries base64 TEXT, encoded once more, so its decoded head // reads as base64 characters. A text column holds the bare base64 of the // nonce, tag and ciphertext, whose decoded head is random bytes: reading // that as an encoded Bytes column would base64-decode it once too often, // and the key that opens it would look like the wrong one. return { keyId: LEGACY_INNER_KEY_ID, form: BASE64_TEXT.test(text) ? "bytes-string" : "string", }; } /** Whether a member name holds inner ciphertext. */ export function isEncryptedMember(name: string): boolean { return name.endsWith("Encrypted") && name.length > "Encrypted".length; } /** * Sections whose restore opens every inner value with this host's keys itself, * and keeps out (and names in the skip report) any value it cannot open. * * The cycle day-logs, custom cycle symptoms, custom mood tags and mood * categories, practitioners, visits and vaccinations are here because a * portable file written before v1.40 carried their free text as the stored * ciphertext, * not as readable text. Such a file is still restorable onto another host: * what that host can open comes back, what it cannot stays out of the row and * is reported. Refusing the whole file over a note would cost every other * part of the account. The trade, stated: a disaster-recovery copy whose ONLY * missing key sits in these sections restores with those values reported as * skipped instead of being refused; any other section needing the key still * refuses it, which in practice is every real copy. */ export const RESTORE_SELF_VERIFIED_SECTIONS: ReadonlySet = new Set([ "cycleDayLogs", "customSymptoms", "customMoodTags", "customMoodTagCategories", "practitioners", "encounters", "vaccinations", ]); /** * One value kept to prove a key opens what it was written under: the * shortest of its section, so the probe of a section the caller will not * write never stands in for one it will. */ export interface KeySample { value: string; form: InnerCiphertextForm; /** * The section it was found in. Absent on a preview stored before samples * were kept per section; such a sample is probed whatever is ignored. */ section?: string; /** The member it sat under, which names the label a binary value needs. */ member?: string; } interface KeyUse { /** How many values need this key. */ count: number; /** The top-level sections they sit in (`measurements`, `appSettings`, …). */ sections: Set; /** The shortest openable value of each section, keyed by section. */ samples: Map; } /** * The associated-data label a binary value under `member` is sealed with, * from the encrypted-column registry: `undefined` for an unlabelled column * (a document's content), null when no binary column has that name or two * with that name use different labels. A null value is not probed: GCM * cannot tell a wrong label from a wrong key, so opening it without the * right one would accuse the key. */ const BINARY_MEMBER_AAD: ReadonlyMap = (() => { const map = new Map(); for (const column of ENCRYPTED_COLUMNS) { if (column.codec !== "binary2" && column.codecField === undefined) { continue; } if (!map.has(column.field)) map.set(column.field, column.aad); else if (map.get(column.field) !== column.aad) map.set(column.field, null); } return map; })(); function binaryAad(member: string | undefined): string | undefined | null { if (member === undefined) return null; const aad = BINARY_MEMBER_AAD.get(member); return aad === undefined && !BINARY_MEMBER_AAD.has(member) ? null : aad; } /** A sample longer than this is not kept; opening it proves nothing more. */ const MAX_SAMPLE_CHARS = 64 * 1024; /** * Collects the key ids a backup's inner ciphertext needs, with the section * each one appears in and one sample per key. */ export class BackupKeyIdCollector { private readonly uses = new Map(); /** Record one inner value found in `section`, under `member`. */ add(section: string, value: string, member?: string): void { const found = innerCiphertextKeyId(value); if (!found) return; let use = this.uses.get(found.keyId); if (!use) { use = { count: 0, sections: new Set(), samples: new Map() }; this.uses.set(found.keyId, use); } use.count += 1; use.sections.add(section); // A sample from a self-verified section proves nothing the restore needs: // a value there that does not open is skipped, not fatal. A binary value // whose label is not known cannot prove anything either. if ( RESTORE_SELF_VERIFIED_SECTIONS.has(section) || value.length > MAX_SAMPLE_CHARS || (found.form === "binary" && binaryAad(member) === null) ) { return; } const kept = use.samples.get(section); if (!kept || value.length < kept.value.length) { use.samples.set(section, { value, form: found.form, section, member }); } } /** * Walk a parsed value and record every `…Encrypted` string in it. `section` * names where it sits in the file; for a whole document, pass nothing and * each top-level member is its own section. */ visit(value: unknown, section?: string): void { if (section === undefined) { if (value && typeof value === "object" && !Array.isArray(value)) { for (const [key, child] of Object.entries(value)) { this.walk(child, key, key); } } return; } this.walk(value, section, null); } private walk(value: unknown, section: string, member: string | null): void { if (typeof value === "string") { if (member !== null && isEncryptedMember(member)) { this.add(section, value, member); } return; } if (Array.isArray(value)) { // Elements of an array under an `…Encrypted` member are ciphertext too. for (const element of value) this.walk(element, section, member); return; } if (value && typeof value === "object") { for (const [key, child] of Object.entries(value)) { this.walk(child, section, key); } } } /** Every key id seen, sorted. */ keyIds(): string[] { return [...this.uses.keys()].sort(); } /** How many values need `keyId`. */ count(keyId: string): number { return this.uses.get(keyId)?.count ?? 0; } /** @internal for `assessBackupKeys`. */ entries(): ReadonlyMap { return this.uses; } /** * What was collected, as plain data, for a copy's stored preview * (`backup-preview.ts`): enough to hold it against this host's keys again * later without reading the copy. */ toStored(): StoredKeyUse[] { return [...this.uses.entries()] .sort(([a], [b]) => a.localeCompare(b)) .map(([keyId, use]) => ({ keyId, count: use.count, sections: [...use.sections].sort(), samples: [...use.samples.values()] .sort((a, b) => (a.section ?? "").localeCompare(b.section ?? "")) .map((sample) => ({ ...sample })), })); } /** The collector {@link toStored} was taken from. */ static fromStored(stored: readonly StoredKeyUse[]): BackupKeyIdCollector { const collector = new BackupKeyIdCollector(); for (const entry of stored) { const samples = new Map(); for (const sample of entry.samples ?? []) { samples.set(sample.section ?? "", { ...sample }); } // A preview stored before samples were kept per section has one. if (entry.sample) samples.set("", { ...entry.sample }); collector.uses.set(entry.keyId, { count: entry.count, sections: new Set(entry.sections), samples, }); } return collector; } } /** One key id a copy needs, as {@link BackupKeyIdCollector.toStored} keeps it. */ export interface StoredKeyUse { keyId: string; count: number; sections: string[]; /** The shortest openable value of each section. */ samples?: KeySample[]; /** The one sample a preview stored before v1.40.1 kept. */ sample?: { value: string; form: InnerCiphertextForm } | null; } export interface BackupKeyAssessment { /** Every key id the file's inner ciphertext was written under. */ keyIds: string[]; /** Needed and not in `ENCRYPTION_KEYS` on this host. */ missing: string[]; /** * Configured, but the sample written under it does not open: the id is * there with different key material behind it. */ unreadable: string[]; /** Values that need a missing or unreadable key. */ affectedValues: number; } /** * Open one sample. The layout is read again from the value rather than taken * from the record: a preview stored by v1.40.0 holds legacy text values * marked as encoded Bytes columns, and must not keep accusing the key. */ function openSample(sample: KeySample) { const form = innerCiphertextKeyId(sample.value)?.form ?? sample.form; switch (form) { case "string": decrypt(sample.value); return; case "bytes-string": decrypt(Buffer.from(sample.value, "base64").toString("utf8")); return; case "binary": { const aad = binaryAad(sample.member); if (aad === null) throw new Error("No label known for a binary sample"); decryptBytes(Buffer.from(sample.value, "base64"), aad); return; } } } function opensAny(samples: readonly KeySample[]): boolean { for (const sample of samples) { try { openSample(sample); return true; } catch { // The next section's sample may still prove the key. } } return false; } /** * Hold what the file needs against what this host has, and open values under * each key to prove the key under that id is the right one. * * The probes come only from sections the verdict counts: one per section, * the shortest. A key is unreadable when none of them opens. One copy is * written by one host at one moment, when each key id had one key material * behind it, so a configured key that opens any of its values is the key the * copy was written with; a single value that does not open is damage to that * value, and refusing the whole restore over it would cost every other part * of the account. * * `ignoreSections` leaves out sections the caller will not write: the * instance settings, when the operator has not asked for them back. The * sections in {@link RESTORE_SELF_VERIFIED_SECTIONS} are always left out, * because their restore checks each value itself. */ export function assessBackupKeys( collector: BackupKeyIdCollector, options: { ignoreSections?: ReadonlySet } = {}, ): BackupKeyAssessment { const ignore = options.ignoreSections ?? new Set(); const keyIds: string[] = []; const missing: string[] = []; const unreadable: string[] = []; let affectedValues = 0; // Read lazily: a file with no inner ciphertext needs no key, and must not // fail on a host (or a test) that has none configured. let configured: Set | null = null; for (const [keyId, use] of [...collector.entries()].sort(([a], [b]) => a.localeCompare(b), )) { if ( [...use.sections].every( (section) => ignore.has(section) || RESTORE_SELF_VERIFIED_SECTIONS.has(section), ) ) { continue; } keyIds.push(keyId); configured ??= new Set(getConfiguredKeyIds()); if (!configured.has(keyId)) { missing.push(keyId); affectedValues += use.count; continue; } const probes = [...use.samples.values()].filter( (sample) => sample.section === undefined || (!ignore.has(sample.section) && !RESTORE_SELF_VERIFIED_SECTIONS.has(sample.section)), ); if (probes.length === 0) continue; if (!opensAny(probes)) { unreadable.push(keyId); affectedValues += use.count; } } return { keyIds, missing, unreadable, affectedValues }; } /** The refusal code every surface answers a key problem with. */ export const BACKUP_KEY_MISSING_CODE = "backup.key.missing"; /** One sentence an operator can act on, naming the key ids. */ export function describeBackupKeyProblem( assessment: BackupKeyAssessment, ): string | null { const parts: string[] = []; if (assessment.missing.length > 0) { const ids = assessment.missing.map((id) => `'${id}'`).join(", "); parts.push( `This backup holds ${assessment.affectedValues} encrypted ${ assessment.affectedValues === 1 ? "value" : "values" } written under encryption key ${ids}, which is not in ENCRYPTION_KEYS on this server. ` + `Add ${assessment.missing.length === 1 ? "that key" : "those keys"} back to ENCRYPTION_KEYS and restart, then try again. Nothing was changed.`, ); } if (assessment.unreadable.length > 0) { const ids = assessment.unreadable.map((id) => `'${id}'`).join(", "); parts.push( `Encryption key ${ids} is configured, but it does not open the values in this backup written under that id: ` + `the key material behind the id differs from the one the backup was written with. Nothing was changed.`, ); } return parts.length > 0 ? parts.join(" ") : null; } /** * Key ids found in backup JSON as it is written, for a copy that never exists * as one document. Fed the text in pieces; a member split across two pieces is * read once the rest of it arrives. */ export interface BackupKeyIdTextScanner { feed(piece: string | Uint8Array): void; keyIds(): string[]; } const MEMBER_VALUE = /"([A-Za-z0-9_]*Encrypted)"\s*:\s*"([A-Za-z0-9+/=._-]{0,64})(")?/g; /** Enough of the previous piece to finish any member that straddles it. */ const CARRY_CHARS = 256; export function createBackupKeyIdTextScanner(): BackupKeyIdTextScanner { const found = new Set(); let carry = ""; return { feed(piece) { // latin1, not utf-8: every character this looks for is ASCII, and a // piece may end in the middle of a multi-byte character. const text = carry + (typeof piece === "string" ? piece : Buffer.from( piece.buffer, piece.byteOffset, piece.byteLength, ).toString("latin1")); MEMBER_VALUE.lastIndex = 0; let match: RegExpExecArray | null; while ((match = MEMBER_VALUE.exec(text)) !== null) { const [, member, value, closed] = match; if (!isEncryptedMember(member)) continue; // A value cut off by the end of the piece is read again next time. if (!closed && value.length < 64) continue; const inner = innerCiphertextKeyId(value); if (inner) found.add(inner.keyId); } carry = text.slice(-CARRY_CHARS); }, keyIds() { return [...found].sort(); }, }; }