Merge branch 'fix/backup-rotation-and-deletion' into release/v1.39.3

# Conflicts:
#	docs/api/openapi.yaml
#	src/lib/jobs/apple-health-import-worker.ts
This commit is contained in:
Marc-André Bombeck committed 2026-09-26 21:48:20 +02:00
commit 4409529c89
74 files changed
+4022 -307

No files matched your search

@@ -0,0 +1,178 @@
/**
* The key ids a backup's inner ciphertext needs, and the verdict against the
* keys this host has.
*
* The values are produced by the real codecs under real keys, then the host's
* key set is changed underneath them, which is exactly what an operator does
* when a rotation is followed by dropping the old key.
*/
import { Buffer } from "node:buffer";
import { afterEach, beforeEach, describe, expect, it } from "vitest";
import { _resetCryptoCacheForTests, encrypt, encryptBytes } from "@/lib/crypto";
import {
assessBackupKeys,
BackupKeyIdCollector,
createBackupKeyIdTextScanner,
describeBackupKeyProblem,
innerCiphertextKeyId,
} from "../backup-key-ids";
const K1 = "11".repeat(32);
const K2 = "22".repeat(32);
const K3 = "33".repeat(32);
function useKeys(keys: Record<string, string>, active: string) {
delete process.env.ENCRYPTION_KEY;
process.env.ENCRYPTION_KEYS = JSON.stringify(keys);
process.env.ENCRYPTION_ACTIVE_KEY_ID = active;
_resetCryptoCacheForTests();
}
const saved = {
key: process.env.ENCRYPTION_KEY,
keys: process.env.ENCRYPTION_KEYS,
active: process.env.ENCRYPTION_ACTIVE_KEY_ID,
};
beforeEach(() => useKeys({ old: K1, cur: K2 }, "old"));
afterEach(() => {
for (const [name, value] of [
["ENCRYPTION_KEY", saved.key],
["ENCRYPTION_KEYS", saved.keys],
["ENCRYPTION_ACTIVE_KEY_ID", saved.active],
] as const) {
if (value === undefined) delete process.env[name];
else process.env[name] = value;
}
_resetCryptoCacheForTests();
});
/** A Bytes column holding the string codec, as a DR payload carries it. */
const bytesString = (plain: string) =>
Buffer.from(encrypt(plain), "utf8").toString("base64");
/** A Bytes column holding the binary codec, as a document's content. */
const binary = (plain: string) =>
encryptBytes(Buffer.from(plain)).toString("base64");
describe("innerCiphertextKeyId", () => {
it("reads the key id of each stored layout", () => {
expect(innerCiphertextKeyId(encrypt("note"))).toEqual({
keyId: "old",
form: "string",
});
expect(innerCiphertextKeyId(bytesString("note"))).toEqual({
keyId: "old",
form: "bytes-string",
});
expect(innerCiphertextKeyId(binary("document"))).toEqual({
keyId: "old",
form: "binary",
});
});
it("reads a value with no key id as the legacy v1 key", () => {
const legacy = Buffer.from("A".repeat(40)).toString("base64");
expect(innerCiphertextKeyId(legacy)?.keyId).toBe("v1");
});
it("ignores an empty value, a short one, and one that is not base64", () => {
expect(innerCiphertextKeyId("")).toBeNull();
expect(innerCiphertextKeyId("yes")).toBeNull();
expect(
innerCiphertextKeyId("not ciphertext at all, and long enough to count!"),
).toBeNull();
});
});
describe("assessBackupKeys", () => {
function fileWrittenUnderOld() {
return {
userId: "u1",
moodEntries: [{ noteEncrypted: bytesString("a mood note") }],
documents: [{ contentEncrypted: binary("%PDF-1.4 …") }],
appSettings: { adminAiKeyEncrypted: encrypt("sk-something") },
};
}
it("passes a file whose keys are all configured and open", () => {
const collector = new BackupKeyIdCollector();
collector.visit(fileWrittenUnderOld());
collector.visit({ notesEncrypted: bytesString("x") }, "measurements");
const verdict = assessBackupKeys(collector);
expect(verdict).toEqual({
keyIds: ["old"],
missing: [],
unreadable: [],
affectedValues: 0,
});
expect(describeBackupKeyProblem(verdict)).toBeNull();
});
it("names a key the file needs and the host no longer has", () => {
const file = fileWrittenUnderOld();
useKeys({ cur: K2 }, "cur");
const collector = new BackupKeyIdCollector();
collector.visit(file);
const verdict = assessBackupKeys(collector);
expect(verdict.missing).toEqual(["old"]);
expect(verdict.affectedValues).toBe(3);
expect(describeBackupKeyProblem(verdict)).toContain("'old'");
});
it("leaves out sections the caller will not write", () => {
useKeys({ old: K1, cur: K2 }, "cur");
const file = {
moodEntries: [{ noteEncrypted: bytesString("fine") }],
appSettings: {
adminAiKeyEncrypted: `gone.${Buffer.alloc(40).toString("base64")}`,
},
};
const collector = new BackupKeyIdCollector();
collector.visit(file);
expect(assessBackupKeys(collector).missing).toEqual(["gone"]);
expect(
assessBackupKeys(collector, { ignoreSections: new Set(["appSettings"]) })
.missing,
).toEqual([]);
});
it("catches an id that is configured with different key material", () => {
const file = fileWrittenUnderOld();
useKeys({ old: K3, cur: K2 }, "cur");
const collector = new BackupKeyIdCollector();
collector.visit(file);
const verdict = assessBackupKeys(collector);
expect(verdict.missing).toEqual([]);
expect(verdict.unreadable).toEqual(["old"]);
});
});
describe("createBackupKeyIdTextScanner", () => {
it("finds every key id in JSON fed in arbitrary pieces", () => {
const early = bytesString("written before the rotation");
useKeys({ old: K1, cur: K2 }, "cur");
const late = binary("written after it");
const json = JSON.stringify({
measurements: [{ id: "m1", notesEncrypted: early }],
documents: [{ contentEncrypted: late, note: "plain" }],
plain: "noteEncrypted",
});
for (const size of [1, 7, 33, 4096]) {
const scanner = createBackupKeyIdTextScanner();
for (let i = 0; i < json.length; i += size) {
scanner.feed(Buffer.from(json.slice(i, i + size), "utf8"));
}
expect(scanner.keyIds()).toEqual(["cur", "old"]);
}
});
it("reads pretty-printed JSON", () => {
const scanner = createBackupKeyIdTextScanner();
scanner.feed(
JSON.stringify({ a: { noteEncrypted: bytesString("x") } }, null, 2),
);
expect(scanner.keyIds()).toEqual(["old"]);
});
});
@@ -0,0 +1,70 @@
/**
* An uploaded backup is counted after the gunzip as well as before it.
* Mutation that must turn this red: drop the inflated count (the bomb reads
* through to the end).
*/
import { gzipSync } from "node:zlib";
import { describe, expect, it } from "vitest";
import {
BackupUploadDecodeError,
decodeBackupUpload,
} from "../backup-upload-decode";
async function drain(
bytes: Buffer,
limits: Parameters<typeof decodeBackupUpload>[1],
): Promise<Buffer> {
async function* source() {
for (let i = 0; i < bytes.length; i += 4096) {
yield bytes.subarray(i, i + 4096);
}
}
const out: Buffer[] = [];
for await (const chunk of decodeBackupUpload(source(), limits)) {
out.push(Buffer.from(chunk));
}
return Buffer.concat(out);
}
describe("decodeBackupUpload", () => {
it("passes plain JSON through and gunzips a gzip file", async () => {
const json = Buffer.from(JSON.stringify({ userId: "u1", n: [1, 2, 3] }));
expect(await drain(json, { compressedLimit: 1024 })).toEqual(json);
expect(await drain(gzipSync(json), { compressedLimit: 1024 })).toEqual(
json,
);
});
it("refuses a gzip that inflates far past its own size", async () => {
const bomb = gzipSync(Buffer.alloc(32 * 1024 * 1024));
expect(bomb.length).toBeLessThan(64 * 1024);
const refused = drain(bomb, {
compressedLimit: 1024 * 1024,
minAllowance: 1024 * 1024,
ratio: 100,
});
await expect(refused).rejects.toBeInstanceOf(BackupUploadDecodeError);
await expect(refused).rejects.toMatchObject({
status: 413,
reason: "inflated_size_exceeded",
});
});
it("refuses a file that is not valid gzip", async () => {
const broken = Buffer.concat([
Buffer.from([0x1f, 0x8b]),
Buffer.from("not gzip"),
]);
await expect(
drain(broken, { compressedLimit: 1024 }),
).rejects.toMatchObject({ status: 422, reason: "invalid_gzip" });
});
it("refuses more bytes than the upload limit", async () => {
await expect(
drain(Buffer.alloc(10_000, 0x20), { compressedLimit: 5_000 }),
).rejects.toMatchObject({ status: 413, reason: "file_size_exceeded" });
});
});
@@ -420,9 +420,15 @@ function recordingTx(
return new Proxy({} as Record<string, unknown>, {
// `$executeRaw` and friends: the restore's per-account lock, and the
// measurement insert, which answer with a row count.
// measurement insert, which answer with a row count. `$queryRaw` is the
// tenant check's read of the schema's foreign keys, which a mock has none
// of (the integration suite runs it against the real catalogue).
get: (_t, model: string) =>
model.startsWith("$") ? async () => 0 : delegate(model),
model === "$queryRaw"
? async () => []
: model.startsWith("$")
? async () => 0
: delegate(model),
});
}
+333
View File
@@ -0,0 +1,333 @@
/**
* Which application keys the ciphertext INSIDE a backup was written under.
*
* A disaster-recovery backup carries every encrypted column the way the
* database holds it: a note, a document, a coach message travel as the stored
* ciphertext, base64-encoded, and a restore writes that ciphertext back
* verbatim. The envelope around the whole copy is sealed separately, and key
* rotation re-seals that envelope, but it cannot reach inside it, and it
* cannot reach a copy in the off-host bucket or a file an operator keeps on a
* disk at all. So a backup taken before a rotation still needs the key the
* rotation retired, and a restore without that key writes rows back that no
* reader can open. Nothing in the restore used to notice: the envelope opened,
* the schema passed, and the damage showed up later as notes that would not
* decrypt.
*
* This module answers the question the restore has to ask first: which key
* ids does this file need, and are they all here. Every encrypted column in a
* backup payload is named `…Encrypted`, and each value is one of:
*
* - the string codec as stored in a text column: `<keyId>.<base64>`;
* - a Bytes column, base64-encoded, holding the string codec as UTF-8;
* - a Bytes column, base64-encoded, holding the binary codec
* (`0x02 | keyIdLen | keyId | iv | tag | ct`);
* - a legacy value with no key id, which only the `v1` key opens.
*
* Two readers share the classification: a walk over a parsed document (the
* restore, the preview, the upload and the restore drill already hold one),
* and a text scanner for a backup that is being written, which never exists
* as a document (`createBackupKeyIdTextScanner`).
*/
import { Buffer } from "node:buffer";
import { decrypt, decryptBytes, getConfiguredKeyIds } from "@/lib/crypto";
/** The key id a value with no key id prefix was written under. */
export const LEGACY_INNER_KEY_ID = "v1";
const KEY_ID = /^[A-Za-z0-9_-]{1,32}$/;
const VERSIONED = /^([A-Za-z0-9_-]{1,32})\.[A-Za-z0-9+/=]+$/;
const BASE64 = /^[A-Za-z0-9+/]+={0,2}$/;
const BASE64_TAIL = /^[A-Za-z0-9+/=]*$/;
const BYTES_CODEC_VERSION = 0x02;
const MIN_CIPHERTEXT_CHARS = 40;
/** How an inner value is laid out, which decides how a sample is opened. */
export type InnerCiphertextForm = "string" | "bytes-string" | "binary";
export interface InnerCiphertext {
keyId: string;
form: InnerCiphertextForm;
}
/**
* The key id one inner ciphertext value needs, or null for something that is
* not ciphertext at all (an empty string). Decodes only a prefix, so a
* multi-megabyte document costs the same as a note.
*/
export function innerCiphertextKeyId(value: string): InnerCiphertext | null {
// The shortest ciphertext either codec writes is 28 bytes (nonce and tag),
// 40 characters as base64: anything shorter is not ciphertext, and reading
// it as a legacy value would demand a key for a field that holds none.
if (value.length < MIN_CIPHERTEXT_CHARS) return null;
const versioned = VERSIONED.exec(value.slice(0, 64));
if (versioned && value.indexOf(".") === versioned[1].length) {
return { keyId: versioned[1], form: "string" };
}
// A Bytes column, base64-encoded. 48 characters decode to 36 bytes, which
// covers the longest header either codec can have (2 + 32 for the binary
// one, 33 for `<keyId>.`).
const head = value.slice(0, 48);
if (!BASE64.test(head)) return null;
const bytes = Buffer.from(head, "base64");
if (bytes.length >= 3 && bytes[0] === BYTES_CODEC_VERSION) {
const keyIdLen = bytes[1];
if (keyIdLen >= 1 && keyIdLen <= 32 && bytes.length >= 2 + keyIdLen) {
const keyId = bytes.subarray(2, 2 + keyIdLen).toString("latin1");
if (KEY_ID.test(keyId)) return { keyId, form: "binary" };
}
}
const text = bytes.toString("latin1");
const dot = text.indexOf(".");
// The rest of the prefix has to read as base64 too, or a legacy value's
// random bytes could pass for `<keyId>.` about once in a thousand values.
if (
dot > 0 &&
KEY_ID.test(text.slice(0, dot)) &&
BASE64_TAIL.test(text.slice(dot + 1))
) {
return { keyId: text.slice(0, dot), form: "bytes-string" };
}
// No key id anywhere: the legacy layout, which `decrypt` opens with `v1`.
return { keyId: LEGACY_INNER_KEY_ID, form: "bytes-string" };
}
/** Whether a member name holds inner ciphertext. */
export function isEncryptedMember(name: string): boolean {
return name.endsWith("Encrypted") && name.length > "Encrypted".length;
}
interface KeyUse {
/** How many values need this key. */
count: number;
/** The top-level sections they sit in (`measurements`, `appSettings`, …). */
sections: Set<string>;
/** The shortest value seen, kept to prove the key actually opens it. */
sample: { value: string; form: InnerCiphertextForm } | null;
}
/** A sample longer than this is not kept; opening it proves nothing more. */
const MAX_SAMPLE_CHARS = 64 * 1024;
/**
* Collects the key ids a backup's inner ciphertext needs, with the section
* each one appears in and one sample per key.
*/
export class BackupKeyIdCollector {
private readonly uses = new Map<string, KeyUse>();
/** Record one inner value found in `section`. */
add(section: string, value: string): void {
const found = innerCiphertextKeyId(value);
if (!found) return;
let use = this.uses.get(found.keyId);
if (!use) {
use = { count: 0, sections: new Set(), sample: null };
this.uses.set(found.keyId, use);
}
use.count += 1;
use.sections.add(section);
if (
value.length <= MAX_SAMPLE_CHARS &&
(!use.sample || value.length < use.sample.value.length)
) {
use.sample = { value, form: found.form };
}
}
/**
* Walk a parsed value and record every `…Encrypted` string in it. `section`
* names where it sits in the file; for a whole document, pass nothing and
* each top-level member is its own section.
*/
visit(value: unknown, section?: string): void {
if (section === undefined) {
if (value && typeof value === "object" && !Array.isArray(value)) {
for (const [key, child] of Object.entries(value)) {
this.walk(child, key, key);
}
}
return;
}
this.walk(value, section, null);
}
private walk(value: unknown, section: string, member: string | null): void {
if (typeof value === "string") {
if (member !== null && isEncryptedMember(member)) {
this.add(section, value);
}
return;
}
if (Array.isArray(value)) {
// Elements of an array under an `…Encrypted` member are ciphertext too.
for (const element of value) this.walk(element, section, member);
return;
}
if (value && typeof value === "object") {
for (const [key, child] of Object.entries(value)) {
this.walk(child, section, key);
}
}
}
/** Every key id seen, sorted. */
keyIds(): string[] {
return [...this.uses.keys()].sort();
}
/** How many values need `keyId`. */
count(keyId: string): number {
return this.uses.get(keyId)?.count ?? 0;
}
/** @internal for `assessBackupKeys`. */
entries(): ReadonlyMap<string, KeyUse> {
return this.uses;
}
}
export interface BackupKeyAssessment {
/** Every key id the file's inner ciphertext was written under. */
keyIds: string[];
/** Needed and not in `ENCRYPTION_KEYS` on this host. */
missing: string[];
/**
* Configured, but the sample written under it does not open: the id is
* there with different key material behind it.
*/
unreadable: string[];
/** Values that need a missing or unreadable key. */
affectedValues: number;
}
function openSample(sample: { value: string; form: InnerCiphertextForm }) {
switch (sample.form) {
case "string":
decrypt(sample.value);
return;
case "bytes-string":
decrypt(Buffer.from(sample.value, "base64").toString("utf8"));
return;
case "binary":
decryptBytes(Buffer.from(sample.value, "base64"));
return;
}
}
/**
* Hold what the file needs against what this host has, and open one value per
* key to prove the key under that id is the right one.
*
* `ignoreSections` leaves out sections the caller will not write: the
* instance settings, when the operator has not asked for them back.
*/
export function assessBackupKeys(
collector: BackupKeyIdCollector,
options: { ignoreSections?: ReadonlySet<string> } = {},
): BackupKeyAssessment {
const ignore = options.ignoreSections ?? new Set<string>();
const keyIds: string[] = [];
const missing: string[] = [];
const unreadable: string[] = [];
let affectedValues = 0;
// Read lazily: a file with no inner ciphertext needs no key, and must not
// fail on a host (or a test) that has none configured.
let configured: Set<string> | null = null;
for (const [keyId, use] of [...collector.entries()].sort(([a], [b]) =>
a.localeCompare(b),
)) {
if ([...use.sections].every((section) => ignore.has(section))) continue;
keyIds.push(keyId);
configured ??= new Set(getConfiguredKeyIds());
if (!configured.has(keyId)) {
missing.push(keyId);
affectedValues += use.count;
continue;
}
if (!use.sample) continue;
try {
openSample(use.sample);
} catch {
unreadable.push(keyId);
affectedValues += use.count;
}
}
return { keyIds, missing, unreadable, affectedValues };
}
/** The refusal code every surface answers a key problem with. */
export const BACKUP_KEY_MISSING_CODE = "backup.key.missing";
/** One sentence an operator can act on, naming the key ids. */
export function describeBackupKeyProblem(
assessment: BackupKeyAssessment,
): string | null {
const parts: string[] = [];
if (assessment.missing.length > 0) {
const ids = assessment.missing.map((id) => `'${id}'`).join(", ");
parts.push(
`This backup holds ${assessment.affectedValues} encrypted ${
assessment.affectedValues === 1 ? "value" : "values"
} written under encryption key ${ids}, which is not in ENCRYPTION_KEYS on this server. ` +
`Add ${assessment.missing.length === 1 ? "that key" : "those keys"} back to ENCRYPTION_KEYS and restart, then try again. Nothing was changed.`,
);
}
if (assessment.unreadable.length > 0) {
const ids = assessment.unreadable.map((id) => `'${id}'`).join(", ");
parts.push(
`Encryption key ${ids} is configured, but it does not open the values in this backup written under that id: ` +
`the key material behind the id differs from the one the backup was written with. Nothing was changed.`,
);
}
return parts.length > 0 ? parts.join(" ") : null;
}
/**
* Key ids found in backup JSON as it is written, for a copy that never exists
* as one document. Fed the text in pieces; a member split across two pieces is
* read once the rest of it arrives.
*/
export interface BackupKeyIdTextScanner {
feed(piece: string | Uint8Array): void;
keyIds(): string[];
}
const MEMBER_VALUE =
/"([A-Za-z0-9_]*Encrypted)"\s*:\s*"([A-Za-z0-9+/=._-]{0,64})(")?/g;
/** Enough of the previous piece to finish any member that straddles it. */
const CARRY_CHARS = 256;
export function createBackupKeyIdTextScanner(): BackupKeyIdTextScanner {
const found = new Set<string>();
let carry = "";
return {
feed(piece) {
// latin1, not utf-8: every character this looks for is ASCII, and a
// piece may end in the middle of a multi-byte character.
const text =
carry +
(typeof piece === "string"
? piece
: Buffer.from(
piece.buffer,
piece.byteOffset,
piece.byteLength,
).toString("latin1"));
MEMBER_VALUE.lastIndex = 0;
let match: RegExpExecArray | null;
while ((match = MEMBER_VALUE.exec(text)) !== null) {
const [, member, value, closed] = match;
if (!isEncryptedMember(member)) continue;
// A value cut off by the end of the piece is read again next time.
if (!closed && value.length < 64) continue;
const inner = innerCiphertextKeyId(value);
if (inner) found.add(inner.keyId);
}
carry = text.slice(-CARRY_CHARS);
},
keyIds() {
return [...found].sort();
},
};
}
+116
View File
@@ -0,0 +1,116 @@
/**
* An uploaded backup file's bytes, gunzipped when they are gzip, with both
* ends counted.
*
* The upload counted what arrived and nothing after the gunzip, so a few
* megabytes of gzip could inflate without limit. The JSON scanner keeps every
* section but the measurements in memory, which turned such a file into an
* out-of-memory kill of the whole app. The inflated bytes are now held to
* {@link INFLATION_RATIO} times the compressed bytes read so far (a real
* backup compresses about tenfold), never less than
* {@link MIN_INFLATED_ALLOWANCE}, and never more than {@link MAX_INFLATED_BYTES}.
*/
import { Readable } from "node:stream";
import { createGunzip } from "node:zlib";
/** How far an uploaded gzip may expand, measured on the bytes read so far. */
export const INFLATION_RATIO = 100;
/** Below this much output the ratio is not what protects the process. */
export const MIN_INFLATED_ALLOWANCE = 256 * 1024 * 1024;
/**
* The largest file an upload may inflate to. A disaster-recovery backup of
* 1.25 million readings is 662 MB of JSON; this leaves room for several times
* that record and stops a bomb long before memory does.
*/
export const MAX_INFLATED_BYTES = 4 * 1024 * 1024 * 1024;
/** Thrown for a file that is too large or not valid gzip. */
export class BackupUploadDecodeError extends Error {
constructor(
readonly status: 413 | 422,
message: string,
readonly reason:
"file_size_exceeded" | "inflated_size_exceeded" | "invalid_gzip",
readonly size?: number,
) {
super(message);
this.name = "BackupUploadDecodeError";
}
}
export interface DecodeLimits {
/** Cap on the bytes as they arrive. */
compressedLimit: number;
ratio?: number;
minAllowance?: number;
maxInflated?: number;
}
/** Bytes → bytes, gunzipped when the file starts with the gzip magic. */
export async function* decodeBackupUpload(
source: AsyncIterable<Uint8Array>,
limits: DecodeLimits,
): AsyncGenerator<Uint8Array> {
const ratio = limits.ratio ?? INFLATION_RATIO;
const minAllowance = limits.minAllowance ?? MIN_INFLATED_ALLOWANCE;
const maxInflated = limits.maxInflated ?? MAX_INFLATED_BYTES;
let seen = 0;
async function* counted() {
for await (const chunk of source) {
seen += chunk.byteLength;
if (seen > limits.compressedLimit) {
throw new BackupUploadDecodeError(
413,
`Upload exceeds ${Math.round(limits.compressedLimit / 1024 / 1024)} MB limit`,
"file_size_exceeded",
seen,
);
}
yield chunk;
}
}
const iterator = counted()[Symbol.asyncIterator]();
const first = await iterator.next();
if (first.done) return;
const rest = (async function* () {
yield first.value;
for (;;) {
const next = await iterator.next();
if (next.done) return;
yield next.value;
}
})();
const head = first.value;
if (!(head.byteLength >= 2 && head[0] === 0x1f && head[1] === 0x8b)) {
yield* rest;
return;
}
const gunzip = Readable.from(rest).pipe(createGunzip());
let inflated = 0;
try {
for await (const chunk of gunzip) {
inflated += (chunk as Buffer).byteLength;
const allowed = Math.min(
maxInflated,
Math.max(minAllowance, seen * ratio),
);
if (inflated > allowed) {
gunzip.destroy();
throw new BackupUploadDecodeError(
413,
`The uploaded file expands to more than ${Math.round(allowed / 1024 / 1024)} MB, which no backup this server wrote does. Nothing was stored.`,
"inflated_size_exceeded",
inflated,
);
}
yield chunk as Buffer;
}
} catch (err) {
if (err instanceof BackupUploadDecodeError) throw err;
throw new BackupUploadDecodeError(
422,
"Uploaded file is not valid gzip",
"invalid_gzip",
);
}
}
+69 -1
View File
@@ -47,6 +47,17 @@ import {
type BackupSource,
type StreamedBackup,
} from "@/lib/export/streamed-backup";
import {
assessBackupKeys,
BACKUP_KEY_MISSING_CODE,
describeBackupKeyProblem,
} from "@/lib/export/backup-key-ids";
import {
assertNoNewForeignReferences,
countForeignReferences,
ForeignReferenceError,
listTenantEdges,
} from "@/lib/export/tenant-boundary";
import { encryptNote } from "@/lib/crypto/note-cipher";
import { encryptToBytes } from "@/lib/ai/coach/bytes-codec";
import { encryptContextToBytes } from "@/lib/labs/biomarker-store";
@@ -240,6 +251,8 @@ export type RestoreFailureCode =
| "owner_not_found"
| "backup.section.missing"
| "document_ciphertext_missing"
| "backup.key.missing"
| "backup.foreign_reference"
| "time_budget"
| "transaction_failed"
| "interrupted"
@@ -475,6 +488,40 @@ export async function restoreBackup(
);
}
// The ciphertext inside the file is written back verbatim, so every key it
// was written under has to be on this host, and has to be the key that
// wrote it. A copy taken before a key rotation still needs the key the
// rotation retired: the rotation re-seals the stored copy's envelope but
// cannot reach inside it. Refused here, above the first delete, because the
// alternative is an account whose notes, documents and coach history come
// back as rows no reader can open. The instance settings count only when
// the operator asked for them back.
const keyVerdict = assessBackupKeys(streamed.keys, {
ignoreSections: restoreInstanceSettings
? new Set<string>()
: new Set(["appSettings"]),
});
const keyProblem = describeBackupKeyProblem(keyVerdict);
if (keyProblem) {
const keyIds = [...keyVerdict.missing, ...keyVerdict.unreadable];
await auditLog("admin.backups.restore.failed", {
userId: input.actorUserId,
ipAddress: input.ipAddress,
details: {
backupId: backup.id,
ownerId,
reason: "key_missing",
missingKeyIds: keyVerdict.missing,
unreadableKeyIds: keyVerdict.unreadable,
affectedValues: keyVerdict.affectedValues,
},
});
return refused(422, BACKUP_KEY_MISSING_CODE, keyProblem, {
errorCode: BACKUP_KEY_MISSING_CODE,
keyIds,
});
}
// Audit the *intent* before the transaction begins so the trail
// describing "an admin is about to restore <user>" survives even
// if the operation crashes midway.
@@ -605,6 +652,17 @@ export async function restoreBackup(
// commit or roll back, then replaces the account again from its own
// file, which is what it was asked to do.
await tx.$executeRaw`SELECT pg_advisory_xact_lock(hashtext(${`backup-restore:${ownerId}`}))`;
// Every reference this restore writes stays inside the account: the
// references that leave it are counted now and again after the last
// write, and any that grew rolls the whole transaction back
// (`tenant-boundary.ts` says why the check is derived from the schema
// rather than written per section).
const tenantEdges = await listTenantEdges(tx);
const foreignBefore = await countForeignReferences(
tx,
ownerId,
tenantEdges,
);
// Declared INSIDE the transaction so a rollback takes the report with
// it. An accumulator that outlived a failed attempt would carry drops
// that never happened into the next one.
@@ -1835,7 +1893,7 @@ export async function restoreBackup(
for (const episode of payload.illnessEpisodes) {
if (episode.parentConditionId) {
await tx.illnessEpisode.update({
where: { id: episode.id },
where: { id: episode.id, userId: ownerId },
data: { parentConditionId: episode.parentConditionId },
});
}
@@ -2288,6 +2346,10 @@ export async function restoreBackup(
environmentCleared.environmentTravelLocations,
ecgRecordings: ecgCleared.ecgRecordings,
};
assertNoNewForeignReferences(
foreignBefore,
await countForeignReferences(tx, ownerId, tenantEdges),
);
progress.sectionsDone = RESTORE_SECTION_STEPS.length;
report("sections", true);
return { cleared, skipped: summarizeRestoreSkips(skips) };
@@ -2316,6 +2378,12 @@ export async function restoreBackup(
},
});
annotate({ meta: { restoreFailReason: verbose } });
if (err instanceof ForeignReferenceError) {
return refused(422, "backup.foreign_reference", err.message, {
errorCode: "backup.foreign_reference",
references: err.edges,
});
}
if (isStoredBackupReadError(err)) {
// The second read, inside the transaction, found the copy replaced or
// altered. The transaction rolled back, so nothing was changed.
+13 -1
View File
@@ -34,6 +34,7 @@ import {
type BackupJsonProducer,
type PackBackupOptions,
} from "@/lib/export/backup-blob";
import { createBackupKeyIdTextScanner } from "@/lib/export/backup-key-ids";
import {
BackupIntegrityError,
newChunkStreamId,
@@ -112,6 +113,11 @@ export async function storeBackupBlob(
await tx.dataBackupChunk.deleteMany({ where: { backupId: row.id } });
const streamId = newChunkStreamId();
// The key ids the copy's content was written under, read from the JSON
// on its way into the pieces. Rotation re-seals the pieces but never
// what is inside them, so this is what tells an operator how long a
// retired key is still needed (`GET /api/admin/encryption/status`).
const keyScanner = createBackupKeyIdTextScanner();
const { chunks, bytes } = await packBackupChunks(
async (sealed, seq) => {
await tx.dataBackupChunk.create({
@@ -120,7 +126,11 @@ export async function storeBackupBlob(
});
},
streamId,
producer,
(write) =>
producer(async (chunk) => {
keyScanner.feed(chunk);
await write(chunk);
}),
options,
);
@@ -135,6 +145,8 @@ export async function storeBackupBlob(
...(input.ownerAfterRead ? { userId: input.ownerAfterRead() } : {}),
chunkCount: chunks,
chunkStreamId: streamId,
innerKeyIds: keyScanner.keyIds(),
innerKeyIdsRecorded: true,
createdAt: new Date(),
},
});
+12
View File
@@ -21,6 +21,7 @@
* check reads). Run it through `parseBackupPayload` for the rest of the
* validation; together that is the whole-file check.
*/
import { BackupKeyIdCollector } from "@/lib/export/backup-key-ids";
import { scanBackupJson } from "@/lib/export/backup-json-scan";
import {
BACKUP_SCHEMA_VERSION,
@@ -51,6 +52,12 @@ export interface StreamedBackup {
measurementCount: number;
/** The file's size in bytes. */
bytes: number;
/**
* The application key ids the file's inner ciphertext was written under,
* gathered on the first read (`assessBackupKeys` holds them against this
* host's keys).
*/
keys: BackupKeyIdCollector;
/**
* Read the file again and hand its measurements over in file order, in
* batches of at most `size`, each row parsed by the element schema.
@@ -93,6 +100,7 @@ export async function readStreamedBackup(
): Promise<StreamedBackup> {
let firstWithoutId: number | null = null;
let bytes = 0;
const keys = new BackupKeyIdCollector();
async function* counted() {
for await (const chunk of source()) {
bytes += chunk.byteLength;
@@ -104,6 +112,7 @@ export async function readStreamedBackup(
onElement: (_key, element, index) => {
const row = parseMeasurement(element, index);
if (!row.id && firstWithoutId === null) firstWithoutId = index;
keys.visit(element, MEASUREMENTS);
options.onMeasurementChecked?.(index + 1);
},
});
@@ -120,6 +129,8 @@ export async function readStreamedBackup(
);
}
keys.visit(document);
const skipKeys = new Set(
Object.keys(document).filter((key) => key !== MEASUREMENTS),
);
@@ -128,6 +139,7 @@ export async function readStreamedBackup(
raw: document,
measurementCount: streamedCounts[MEASUREMENTS] ?? 0,
bytes,
keys,
async forEachMeasurementBatch(size, fn) {
let batch: BackupMeasurement[] = [];
await scanBackupJson(source(), {
+178
View File
@@ -0,0 +1,178 @@
/**
* Every reference a restore writes has to stay inside the account it writes.
*
* A backup file names ids: a custom mood tag names its category, an illness
* names its parent condition, a link names the document and the episode it
* joins. The restore writes those ids back as given, and a foreign key only
* proves the row they point at exists, not whose it is. So a file edited by
* hand, or an upload whose owner is simply what the file says it is, could
* attach the restored account's rows to rows of another account on the same
* host: a custom tag filed under somebody else's category, a note linked to
* somebody else's document. Several of the restore's writes (the tag and
* category upserts keyed on a globally unique `key`, for one) never even look
* at the owner.
*
* Checking each write site by hand is the approach that keeps missing one, so
* the check is derived from the schema instead. Every single-column foreign key
* is read from the database catalogue, and two shapes are held:
*
* - a table with `user_id` pointing at another table with `user_id`: a row of
* the restored account may not point at a row another account owns;
* - a link table without `user_id` joining two such tables: a link whose one
* end is the restored account's may not have its other end in another
* account.
*
* A NULL owner on the far end is a shared catalogue row (a seeded tag) and is
* allowed. The counts are taken twice inside the restore transaction, before
* the first delete and after the last write, and the restore is refused when
* any count went up: the host may have legitimate cross-account links of its
* own (a shared record), and the question is only whether THIS restore added
* one. Refusing throws, which rolls the whole transaction back.
*/
import type { Prisma } from "@/generated/prisma/client";
type Db = Pick<Prisma.TransactionClient, "$queryRawUnsafe" | "$queryRaw">;
/** One foreign key, or one pair of them through a link table. */
export interface TenantEdge {
/** Readable name for the refusal: `child.column -> parent`. */
name: string;
/** The counting query; `$1` is the owner id. */
sql: string;
}
/**
* Identifiers read from the catalogue are spliced into SQL, so each is held
* to this before use (the whitelist-splice rule for `$queryRawUnsafe`).
*/
const IDENT = /^[a-z_][a-z0-9_]*$/;
interface ForeignKeyRow {
child: string;
child_col: string;
parent: string;
parent_col: string;
child_owned: boolean;
parent_owned: boolean;
}
/** Thrown inside the restore transaction; rolls it back. */
export class ForeignReferenceError extends Error {
readonly edges: string[];
constructor(edges: string[]) {
super(
`The backup references records that belong to another account (${edges.join(", ")}). Nothing was changed.`,
);
this.name = "ForeignReferenceError";
this.edges = edges;
}
}
export async function listTenantEdges(db: Db): Promise<TenantEdge[]> {
const rows = await db.$queryRaw<ForeignKeyRow[]>`
SELECT
child.relname AS child,
ca.attname AS child_col,
parent.relname AS parent,
pa.attname AS parent_col,
EXISTS (
SELECT 1 FROM pg_attribute x
WHERE x.attrelid = c.conrelid AND x.attname = 'user_id'
AND NOT x.attisdropped
) AS child_owned,
EXISTS (
SELECT 1 FROM pg_attribute x
WHERE x.attrelid = c.confrelid AND x.attname = 'user_id'
AND NOT x.attisdropped
) AS parent_owned
FROM pg_constraint c
JOIN pg_class child ON child.oid = c.conrelid
JOIN pg_class parent ON parent.oid = c.confrelid
JOIN pg_namespace n ON n.oid = child.relnamespace
JOIN pg_attribute ca ON ca.attrelid = c.conrelid AND ca.attnum = c.conkey[1]
JOIN pg_attribute pa ON pa.attrelid = c.confrelid AND pa.attnum = c.confkey[1]
WHERE c.contype = 'f'
AND array_length(c.conkey, 1) = 1
AND n.nspname = current_schema()
ORDER BY child.relname, ca.attname
`;
const safe = rows.filter((row) =>
[row.child, row.child_col, row.parent, row.parent_col].every((id) =>
IDENT.test(id),
),
);
if (safe.length !== rows.length) {
// Never silently narrower than the schema: a check that quietly skips a
// table is the kind that stays green because it matched nothing.
throw new Error(
"Tenant check: a foreign key has an identifier outside [a-z0-9_]",
);
}
const edges: TenantEdge[] = [];
for (const fk of safe) {
if (!fk.child_owned || !fk.parent_owned) continue;
edges.push({
name: `${fk.child}.${fk.child_col} -> ${fk.parent}`,
sql:
`SELECT count(*)::int AS n FROM "${fk.child}" c ` +
`JOIN "${fk.parent}" p ON p."${fk.parent_col}" = c."${fk.child_col}" ` +
`WHERE c.user_id = $1 AND p.user_id IS NOT NULL AND p.user_id <> $1`,
});
}
// Link tables: no owner of their own, two or more keys into owned tables.
const byChild = new Map<string, ForeignKeyRow[]>();
for (const fk of safe) {
if (fk.child_owned || !fk.parent_owned) continue;
const list = byChild.get(fk.child) ?? [];
list.push(fk);
byChild.set(fk.child, list);
}
for (const [child, fks] of byChild) {
if (fks.length < 2) continue;
for (const near of fks) {
for (const far of fks) {
if (near === far) continue;
edges.push({
name: `${child}.${near.child_col} / ${child}.${far.child_col} -> ${far.parent}`,
sql:
`SELECT count(*)::int AS n FROM "${child}" l ` +
`JOIN "${near.parent}" a ON a."${near.parent_col}" = l."${near.child_col}" ` +
`JOIN "${far.parent}" b ON b."${far.parent_col}" = l."${far.child_col}" ` +
`WHERE a.user_id = $1 AND b.user_id IS NOT NULL AND b.user_id <> $1`,
});
}
}
}
return edges;
}
/** How many references from `ownerId`'s rows leave the account, per edge. */
export async function countForeignReferences(
db: Db,
ownerId: string,
edges: readonly TenantEdge[],
): Promise<Map<string, number>> {
const counts = new Map<string, number>();
for (const edge of edges) {
const [row] = await db.$queryRawUnsafe<Array<{ n: number }>>(
edge.sql,
ownerId,
);
counts.set(edge.name, row?.n ?? 0);
}
return counts;
}
/** Throw when any edge carries more foreign references than it did before. */
export function assertNoNewForeignReferences(
before: ReadonlyMap<string, number>,
after: ReadonlyMap<string, number>,
): void {
const grown = [...after]
.filter(([name, n]) => n > (before.get(name) ?? 0))
.map(([name]) => name);
if (grown.length > 0) throw new ForeignReferenceError(grown);
}